From f452a2b3e4e4279b27594a8ddb66525442d59227 Mon Sep 17 00:00:00 2001 From: Ondřej Surý Date: Fri, 19 Aug 2011 10:22:38 +0200 Subject: Imported Upstream version 5.3.7 --- ext/sqlite3/libsqlite/sqlite3.c | 160589 +++++++++++++++++++------------------ 1 file changed, 83026 insertions(+), 77563 deletions(-) (limited to 'ext/sqlite3/libsqlite/sqlite3.c') diff --git a/ext/sqlite3/libsqlite/sqlite3.c b/ext/sqlite3/libsqlite/sqlite3.c index 51b0a6a58..d187e66d2 100644 --- a/ext/sqlite3/libsqlite/sqlite3.c +++ b/ext/sqlite3/libsqlite/sqlite3.c @@ -4,8 +4,8 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.4. By combining all the individual C code files into this -** single large file, the entire code can be compiled as a one translation +** version 3.7.7.1. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements ** of 5% or more are commonly seen when SQLite is compiled as a single @@ -204,7 +204,7 @@ /* ** The maximum number of attached databases. This must be between 0 -** and 30. The upper bound on 30 is because a 32-bit integer bitmap +** and 62. The upper bound on 62 is because a 64-bit integer bitmap ** is used internally to track attached databases. */ #ifndef SQLITE_MAX_ATTACHED @@ -654,9 +654,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.4" -#define SQLITE_VERSION_NUMBER 3007004 -#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45" +#define SQLITE_VERSION "3.7.7.1" +#define SQLITE_VERSION_NUMBER 3007007 +#define SQLITE_SOURCE_ID "2011-06-28 17:39:05 af0d91adf497f5f36ec3813f04235a6e195a605f" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -857,7 +857,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row ** coming out of the evaluated SQL statements. ^The 4th argument to -** to sqlite3_exec() is relayed through to the 1st argument of each +** sqlite3_exec() is relayed through to the 1st argument of each ** callback invocation. ^If the callback pointer to sqlite3_exec() ** is NULL, then no callback is ever invoked and result rows are ** ignored. @@ -922,7 +922,8 @@ SQLITE_API int sqlite3_exec( ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes] +** See also: [SQLITE_IOERR_READ | extended result codes], +** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -937,7 +938,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ #define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ #define SQLITE_CORRUPT 11 /* The database disk image is malformed */ -#define SQLITE_NOTFOUND 12 /* NOT USED. Table or record not found */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ #define SQLITE_FULL 13 /* Insertion failed because database is full */ #define SQLITE_CANTOPEN 14 /* Unable to open the database file */ #define SQLITE_PROTOCOL 15 /* Database lock protocol error */ @@ -999,17 +1000,21 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations ** ** These bit values are intended for use in the ** 3rd parameter to the [sqlite3_open_v2()] interface and -** in the 4th parameter to the xOpen method of the -** [sqlite3_vfs] object. +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ @@ -1017,6 +1022,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ #define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -1030,6 +1036,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +/* Reserved: 0x00F00000 */ + /* ** CAPI3REF: Device Characteristics ** @@ -1125,17 +1133,18 @@ struct sqlite3_file { /* ** CAPI3REF: OS Interface File Virtual Methods Object ** -** Every file opened by the [sqlite3_vfs] xOpen method populates an +** Every file opened by the [sqlite3_vfs.xOpen] method populates an ** [sqlite3_file] object (or, more commonly, a subclass of the ** [sqlite3_file] object) with a pointer to an instance of this object. ** This object defines the methods used to perform various operations ** against the open file represented by the [sqlite3_file] object. ** -** If the xOpen method sets the sqlite3_file.pMethods element +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element ** to a non-NULL pointer, then the sqlite3_io_methods.xClose method -** may be invoked even if the xOpen reported that it failed. The -** only way to prevent a call to xClose following a failed xOpen -** is for the xOpen to set the sqlite3_file.pMethods element to NULL. +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. ** ** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or ** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). @@ -1169,7 +1178,9 @@ struct sqlite3_file { ** core reserves all opcodes less than 100 for its own use. ** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available. ** Applications that define a custom xFileControl method should use opcodes -** greater than 100 to avoid conflicts. +** greater than 100 to avoid conflicts. VFS implementations should +** return [SQLITE_NOTFOUND] for file control opcodes that they do not +** recognize. ** ** The xSectorSize() method returns the sector size of the ** device that underlies the file. The sector size is the @@ -1262,6 +1273,21 @@ struct sqlite3_io_methods { ** for the nominated database. Allocating database file space in large ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. +** +** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with a particular database +** connection. See the [sqlite3_file_control()] documentation for +** additional information. +** +** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by +** SQLite and sent to all VFSes in place of a call to the xSync method +** when the database connection has [PRAGMA synchronous] set to OFF.)^ +** Some specialized VFSes need this signal in order to operate correctly +** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most +** VFSes do not need this signal and should silently ignore this opcode. +** Applications should not call [sqlite3_file_control()] with this +** opcode as doing so may disrupt the operation of the specialized VFSes +** that do require it. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -1270,6 +1296,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_SIZE_HINT 5 #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 /* @@ -1289,7 +1316,8 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** An instance of the sqlite3_vfs object defines the interface between ** the SQLite core and the underlying operating system. The "vfs" -** in the name of the object stands for "virtual file system". +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. ** ** The value of the iVersion field is initially 1 but may be larger in ** future versions of SQLite. Additional fields may be appended to this @@ -1318,6 +1346,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** +** [[sqlite3_vfs.xOpen]] ** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained ** from xFullPathname() with an optional suffix added. @@ -1395,6 +1424,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** +** [[sqlite3_vfs.xAccess]] ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] @@ -1419,16 +1449,29 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** method returns a Julian Day Number for the current date and time as ** a floating point value. ** ^The xCurrentTimeInt64() method returns, as an integer, the Julian -** Day Number multipled by 86400000 (the number of milliseconds in +** Day Number multiplied by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current ** date and time if that method is available (if iVersion is 2 or ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. +** +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** are not used by the SQLite core. These optional interfaces are provided +** by some VFSes to facilitate testing of the VFS code. By overriding +** system calls with functions under its control, a test program can +** simulate faults and error conditions that would otherwise be difficult +** or impossible to induce. The set of system calls that can be overridden +** varies from one VFS to another, and from one version of the same VFS to the +** next. Applications that use these interfaces must be prepared for any +** or all of these interfaces to be NULL or for their behavior to change +** from one release to the next. Applications must not attempt to access +** any of these methods if the iVersion of the VFS is less than 3. */ typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); struct sqlite3_vfs { - int iVersion; /* Structure version number (currently 2) */ + int iVersion; /* Structure version number (currently 3) */ int szOsFile; /* Size of subclassed sqlite3_file */ int mxPathname; /* Maximum file pathname length */ sqlite3_vfs *pNext; /* Next registered VFS */ @@ -1454,6 +1497,13 @@ struct sqlite3_vfs { int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); /* ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. ** New fields may be appended in figure versions. The iVersion ** value will increment whenever this happens. */ @@ -1621,9 +1671,9 @@ SQLITE_API int sqlite3_os_end(void); ** implementation of an application-defined [sqlite3_os_init()]. ** ** The first argument to sqlite3_config() is an integer -** [SQLITE_CONFIG_SINGLETHREAD | configuration option] that determines +** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments -** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option] +** vary depending on the [configuration option] ** in the first argument. ** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. @@ -1638,17 +1688,12 @@ SQLITE_API int sqlite3_config(int, ...); ** The sqlite3_db_config() interface is used to make configuration ** changes to a [database connection]. The interface is similar to ** [sqlite3_config()] except that the changes apply to a single -** [database connection] (specified in the first argument). The -** sqlite3_db_config() interface should only be used immediately after -** the database connection is created using [sqlite3_open()], -** [sqlite3_open16()], or [sqlite3_open_v2()]. +** [database connection] (specified in the first argument). ** ** The second argument to sqlite3_db_config(D,V,...) is the -** configuration verb - an integer code that indicates what -** aspect of the [database connection] is being configured. -** The only choice for this value is [SQLITE_DBCONFIG_LOOKASIDE]. -** New verbs are likely to be added in future releases of SQLite. -** Additional arguments depend on the verb. +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** that indicates what aspect of the [database connection] is being configured. +** Subsequent arguments vary depending on the configuration verb. ** ** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if ** the call is considered successful. @@ -1738,6 +1783,7 @@ struct sqlite3_mem_methods { /* ** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} ** ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. @@ -1750,7 +1796,7 @@ struct sqlite3_mem_methods { ** is invoked. ** **
-**
SQLITE_CONFIG_SINGLETHREAD
+** [[SQLITE_CONFIG_SINGLETHREAD]]
SQLITE_CONFIG_SINGLETHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Single-thread. In other words, it disables ** all mutexing and puts SQLite into a mode where it can only be used @@ -1761,7 +1807,7 @@ struct sqlite3_mem_methods { ** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD ** configuration option.
** -**
SQLITE_CONFIG_MULTITHREAD
+** [[SQLITE_CONFIG_MULTITHREAD]]
SQLITE_CONFIG_MULTITHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Multi-thread. In other words, it disables ** mutexing on [database connection] and [prepared statement] objects. @@ -1775,7 +1821,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_MULTITHREAD configuration option.
** -**
SQLITE_CONFIG_SERIALIZED
+** [[SQLITE_CONFIG_SERIALIZED]]
SQLITE_CONFIG_SERIALIZED
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Serialized. In other words, this option enables ** all mutexes including the recursive @@ -1791,7 +1837,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_SERIALIZED configuration option.
** -**
SQLITE_CONFIG_MALLOC
+** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The argument specifies ** alternative low-level memory allocation routines to be used in place of @@ -1799,7 +1845,7 @@ struct sqlite3_mem_methods { ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
** -**
SQLITE_CONFIG_GETMALLOC
+** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ @@ -1807,7 +1853,7 @@ struct sqlite3_mem_methods { ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
** -**
SQLITE_CONFIG_MEMSTATUS
+** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
**
^This option takes single argument of type int, interpreted as a ** boolean, which enables or disables the collection of memory allocation ** statistics. ^(When memory allocation statistics are disabled, the @@ -1823,10 +1869,10 @@ struct sqlite3_mem_methods { ** allocation statistics are disabled by default. **
** -**
SQLITE_CONFIG_SCRATCH
+** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
**
^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte -** aligned memory buffer from which the scrach allocations will be +** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz ** argument must be a multiple of 16. @@ -1839,9 +1885,9 @@ struct sqlite3_mem_methods { ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
** -**
SQLITE_CONFIG_PAGECACHE
+** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
**
^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implemenation. +** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. ** There are three arguments to this option: A pointer to 8-byte aligned @@ -1860,7 +1906,7 @@ struct sqlite3_mem_methods { ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
** -**
SQLITE_CONFIG_HEAP
+** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
**
^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. @@ -1873,9 +1919,11 @@ struct sqlite3_mem_methods { ** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte -** boundary or subsequent behavior of SQLite will be undefined.
+** boundary or subsequent behavior of SQLite will be undefined. +** The minimum allocation size is capped at 2^12. Reasonable values +** for the minimum allocation size are 2^5 through 2^8. ** -**
SQLITE_CONFIG_MUTEX
+** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The argument specifies ** alternative low-level mutex routines to be used in place @@ -1887,7 +1935,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_GETMUTEX
+** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] @@ -1900,7 +1948,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_LOOKASIDE
+** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
**
^(This option takes two arguments that determine the default ** memory allocation for the lookaside memory allocator on each ** [database connection]. The first argument is the @@ -1910,18 +1958,18 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
** -**
SQLITE_CONFIG_PCACHE
+** [[SQLITE_CONFIG_PCACHE]]
SQLITE_CONFIG_PCACHE
**
^(This option takes a single argument which is a pointer to ** an [sqlite3_pcache_methods] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
** -**
SQLITE_CONFIG_GETPCACHE
+** [[SQLITE_CONFIG_GETPCACHE]]
SQLITE_CONFIG_GETPCACHE
**
^(This option takes a single argument which is a pointer to an ** [sqlite3_pcache_methods] object. SQLite copies of the current ** page cache implementation into that object.)^
** -**
SQLITE_CONFIG_LOG
+** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
**
^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is @@ -1939,6 +1987,18 @@ struct sqlite3_mem_methods { ** In a multi-threaded application, the application-defined logger ** function must be threadsafe.
** +** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI +**
This option takes a single argument of type int. If non-zero, then +** URI handling is globally enabled. If the parameter is zero, then URI handling +** is globally disabled. If URI handling is globally enabled, all filenames +** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined. **
*/ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1957,6 +2017,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1976,7 +2037,7 @@ struct sqlite3_mem_methods { **
^This option takes three additional arguments that determine the ** [lookaside memory allocator] configuration for the [database connection]. ** ^The first argument (the third parameter to [sqlite3_db_config()] is a -** pointer to an memory buffer to use for lookaside memory. +** pointer to a memory buffer to use for lookaside memory. ** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb ** may be NULL in which case SQLite will allocate the ** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the @@ -1994,9 +2055,31 @@ struct sqlite3_mem_methods { ** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^
** +**
SQLITE_DBCONFIG_ENABLE_FKEY
+**
^This option is used to enable or disable the enforcement of +** [foreign key constraints]. There should be two additional arguments. +** The first argument is an integer which is 0 to disable FK enforcement, +** positive to enable FK enforcement or negative to leave FK enforcement +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether FK enforcement is off or on +** following this call. The second parameter may be a NULL pointer, in +** which case the FK enforcement setting is not reported back.
+** +**
SQLITE_DBCONFIG_ENABLE_TRIGGER
+**
^This option is used to enable or disable [CREATE TRIGGER | triggers]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable triggers, +** positive to enable triggers or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether triggers are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the trigger setting is not reported back.
+** ** */ -#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ /* @@ -2020,13 +2103,17 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); ** ** ^This routine returns the [rowid] of the most recent ** successful [INSERT] into the database from the [database connection] -** in the first argument. ^If no successful [INSERT]s +** in the first argument. ^As of SQLite version 3.7.7, this routines +** records the last insert rowid of both ordinary tables and [virtual tables]. +** ^If no successful [INSERT]s ** have ever occurred on that database connection, zero is returned. ** -** ^(If an [INSERT] occurs within a trigger, then the [rowid] of the inserted -** row is returned by this routine as long as the trigger is running. -** But once the trigger terminates, the value returned by this routine -** reverts to the last value inserted before the trigger fired.)^ +** ^(If an [INSERT] occurs within a trigger or within a [virtual table] +** method, then this routine will return the [rowid] of the inserted +** row as long as the trigger or virtual table method is running. +** But once the trigger or virtual table method ends, the value returned +** by this routine reverts to what it was before the trigger or virtual +** table method began.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2389,7 +2476,7 @@ SQLITE_API void sqlite3_free_table(char **result); ** NULL pointer if [sqlite3_malloc()] is unable to allocate enough ** memory to hold the resulting string. ** -** ^(In sqlite3_snprintf() routine is similar to "snprintf()" from +** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from ** the standard C library. The result is written into the ** buffer supplied as the second parameter whose size is given by ** the first parameter. Note that the order of the @@ -2408,6 +2495,8 @@ SQLITE_API void sqlite3_free_table(char **result); ** the zero terminator. So the longest string that can be completely ** written will be n-1 characters. ** +** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). +** ** These routines all implement some additional formatting ** options that are useful for constructing SQL statements. ** All of the usual printf() formatting options apply. In addition, there @@ -2471,6 +2560,7 @@ SQLITE_API void sqlite3_free_table(char **result); SQLITE_API char *sqlite3_mprintf(const char*,...); SQLITE_API char *sqlite3_vmprintf(const char*, va_list); SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); /* ** CAPI3REF: Memory Allocation Subsystem @@ -2595,7 +2685,7 @@ SQLITE_API void sqlite3_randomness(int N, void *P); /* ** CAPI3REF: Compile-Time Authorization Callbacks ** -** ^This routine registers a authorizer callback with a particular +** ^This routine registers an authorizer callback with a particular ** [database connection], supplied in the first argument. ** ^The authorizer callback is invoked as SQL statements are being compiled ** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()], @@ -2686,6 +2776,9 @@ SQLITE_API int sqlite3_set_authorizer( ** to signal SQLite whether or not the action is permitted. See the ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. +** +** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] +** from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2808,7 +2901,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection ** -** ^These routines open an SQLite database file whose name is given by the +** ^These routines open an SQLite database file as specified by the ** filename argument. ^The filename argument is interpreted as UTF-8 for ** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte ** order for sqlite3_open16(). ^(A [database connection] handle is usually @@ -2835,7 +2928,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2() can take one of ** the following three values, optionally combined with the ** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], -** and/or [SQLITE_OPEN_PRIVATECACHE] flags:)^ +** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ ** **
** ^(
[SQLITE_OPEN_READONLY]
@@ -2848,15 +2941,14 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** case the database must already exist, otherwise an error is returned.)^ ** ** ^(
[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
-**
The database is opened for reading and writing, and is creates it if +**
The database is opened for reading and writing, and is created if ** it does not already exist. This is the behavior that is always used for ** sqlite3_open() and sqlite3_open16().
)^ **
** ** If the 3rd parameter to sqlite3_open_v2() is not one of the -** combinations shown above or one of the combinations shown above combined -** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, +** combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2871,6 +2963,11 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not ** participate in [shared cache mode] even if it is enabled. ** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** ** ^If the filename is ":memory:", then a private, temporary in-memory database ** is created for the connection. ^This in-memory database will vanish when ** the database connection is closed. Future versions of SQLite might @@ -2883,10 +2980,111 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** on-disk database will be created. ^This private database will be ** automatically deleted as soon as the database connection is closed. ** -** ^The fourth parameter to sqlite3_open_v2() is the name of the -** [sqlite3_vfs] object that defines the operating system interface that -** the new database connection should use. ^If the fourth parameter is -** a NULL pointer then the default [sqlite3_vfs] object is used. +** [[URI filenames in sqlite3_open()]]

URI Filenames

+** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the fourth argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** As of SQLite version 3.7.7, URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^On windows, the first component of an absolute path +** is a drive specification (e.g. "C:"). +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite interprets the following three query parameters: +** +** +** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

URI filename examples

+** +** +**
URI filenames Results +**
file:data.db +** Open the file "data.db" in the current directory. +**
file:/home/fred/data.db
+** file:///home/fred/data.db
+** file://localhost/home/fred/data.db
+** Open the database file "/home/fred/data.db". +**
file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
+** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
file:/home/fred/data.db?vfs=unix-nolock +** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +**
file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +**
+** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. ** ** Note to Windows users: The encoding used for the filename argument ** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever @@ -2909,6 +3107,26 @@ SQLITE_API int sqlite3_open_v2( const char *zVfs /* Name of VFS module to use */ ); +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +** +** If the zFilename argument to this function is not a pointer that SQLite +** passed into the xOpen VFS method, then the behavior of this routine +** is undefined and probably undesirable. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); + + /* ** CAPI3REF: Error Codes And Messages ** @@ -3024,43 +3242,45 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** Additional information is available at [limits | Limits in SQLite]. ** **
-** ^(
SQLITE_LIMIT_LENGTH
+** [[SQLITE_LIMIT_LENGTH]] ^(
SQLITE_LIMIT_LENGTH
**
The maximum size of any string or BLOB or table row, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_SQL_LENGTH
+** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
SQLITE_LIMIT_SQL_LENGTH
**
The maximum length of an SQL statement, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_COLUMN
+** [[SQLITE_LIMIT_COLUMN]] ^(
SQLITE_LIMIT_COLUMN
**
The maximum number of columns in a table definition or in the ** result set of a [SELECT] or the maximum number of columns in an index ** or in an ORDER BY or GROUP BY clause.
)^ ** -** ^(
SQLITE_LIMIT_EXPR_DEPTH
+** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
SQLITE_LIMIT_EXPR_DEPTH
**
The maximum depth of the parse tree on any expression.
)^ ** -** ^(
SQLITE_LIMIT_COMPOUND_SELECT
+** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
SQLITE_LIMIT_COMPOUND_SELECT
**
The maximum number of terms in a compound SELECT statement.
)^ ** -** ^(
SQLITE_LIMIT_VDBE_OP
+** [[SQLITE_LIMIT_VDBE_OP]] ^(
SQLITE_LIMIT_VDBE_OP
**
The maximum number of instructions in a virtual machine program ** used to implement an SQL statement. This limit is not currently ** enforced, though that might be added in some future release of ** SQLite.
)^ ** -** ^(
SQLITE_LIMIT_FUNCTION_ARG
+** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
SQLITE_LIMIT_FUNCTION_ARG
**
The maximum number of arguments on a function.
)^ ** -** ^(
SQLITE_LIMIT_ATTACHED
+** [[SQLITE_LIMIT_ATTACHED]] ^(
SQLITE_LIMIT_ATTACHED
**
The maximum number of [ATTACH | attached databases].)^
** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] ** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH
**
The maximum length of the pattern argument to the [LIKE] or ** [GLOB] operators.
)^ ** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] ** ^(
SQLITE_LIMIT_VARIABLE_NUMBER
**
The maximum index number of any [parameter] in an SQL statement.)^ ** -** ^(
SQLITE_LIMIT_TRIGGER_DEPTH
+** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
**
The maximum depth of recursion for triggers.
)^ **
*/ @@ -3198,13 +3418,30 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** CAPI3REF: Determine If An SQL Statement Writes The Database ** ** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if -** the [prepared statement] X is [SELECT] statement and false (zero) if -** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE], -** [ALTER], or [REINDEX] statement. -** If X is a NULL pointer or any other kind of statement, including but -** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE], -** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is -** undefined. +** and only if the [prepared statement] X makes no direct changes to +** the content of the database file. +** +** Note that [application-defined SQL functions] or +** [virtual tables] might change the database indirectly as a side effect. +** ^(For example, if an application defines a function "eval()" that +** calls [sqlite3_exec()], then the following SQL statement would +** change the database file through side-effects: +** +** 
+**    SELECT eval('DELETE FROM t1') FROM t2;
+**
+** +** But because the [SELECT] statement does not change the database file +** directly, sqlite3_stmt_readonly() would still return true.)^ +** +** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK], +** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true, +** since the statements themselves do not actually modify the database but +** rather they control the timing of when other statements modify the +** database. ^The [ATTACH] and [DETACH] statements also cause +** sqlite3_stmt_readonly() to return true since, while those statements +** change the configuration of a database connection, they do not make +** changes to the content of the database files on disk. */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); @@ -3224,7 +3461,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); ** whether or not it requires a protected sqlite3_value. ** ** The terms "protected" and "unprotected" refer to whether or not -** a mutex is held. A internal mutex is held for a protected +** a mutex is held. An internal mutex is held for a protected ** sqlite3_value object but no mutex is held for an unprotected ** sqlite3_value object. If SQLite is compiled to be single-threaded ** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) @@ -3448,7 +3685,9 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); ** column number. ^The leftmost column is number 0. ** ** ^The returned string pointer is valid until either the [prepared statement] -** is destroyed by [sqlite3_finalize()] or until the next call to +** is destroyed by [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the next call to ** sqlite3_column_name() or sqlite3_column_name16() on the same column. ** ** ^If sqlite3_malloc() fails during the processing of either routine @@ -3474,7 +3713,9 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** the database name, the _table_ routines return the table name, and ** the origin_ routines return the column name. ** ^The returned string is valid until the [prepared statement] is destroyed -** using [sqlite3_finalize()] or until the same information is requested +** using [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the same information is requested ** again in a different encoding. ** ** ^The names returned are the original un-aliased names of the @@ -3568,7 +3809,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** ^[SQLITE_BUSY] means that the database engine was unable to acquire the ** database locks it needs to do its job. ^If the statement is a [COMMIT] ** or occurs outside of an explicit transaction, then you can retry the -** statement. If the statement is not a [COMMIT] and occurs within a +** statement. If the statement is not a [COMMIT] and occurs within an ** explicit transaction then you should rollback the transaction before ** continuing. ** @@ -3598,13 +3839,17 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** be the case that the same database connection is being used by two or ** more threads at the same moment in time. ** -** For all versions of SQLite up to and including 3.6.23.1, it was required -** after sqlite3_step() returned anything other than [SQLITE_ROW] that -** [sqlite3_reset()] be called before any subsequent invocation of -** sqlite3_step(). Failure to invoke [sqlite3_reset()] in this way would -** result in an [SQLITE_MISUSE] return from sqlite3_step(). But after -** version 3.6.23.1, sqlite3_step() began calling [sqlite3_reset()] -** automatically in this circumstance rather than returning [SQLITE_MISUSE]. +** For all versions of SQLite up to and including 3.6.23.1, a call to +** [sqlite3_reset()] was required after sqlite3_step() returned anything +** other than [SQLITE_ROW] before any subsequent invocation of +** sqlite3_step(). Failure to reset the prepared statement using +** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from +** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began +** calling [sqlite3_reset()] automatically in this circumstance rather +** than returning [SQLITE_MISUSE]. This is not considered a compatibility +** break because any application that ever receives an SQLITE_MISUSE error +** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option +** can be used to restore the legacy behavior. ** ** Goofy Interface Alert: In the legacy interface, the sqlite3_step() ** API always returns a generic error code, [SQLITE_ERROR], following any @@ -3843,7 +4088,7 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the most recent evaluation of the statement encountered no errors or +** ^If the most recent evaluation of the statement encountered no errors ** or if the statement is never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or @@ -3902,7 +4147,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** are used to add SQL functions or aggregates or to redefine the behavior ** of existing SQL functions or aggregates. The only differences between ** these routines are the text encoding expected for -** the the second parameter (the name of the function being created) +** the second parameter (the name of the function being created) ** and the presence or absence of a destructor callback for ** the application data pointer. ** @@ -3941,16 +4186,16 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc ** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep ** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL poiners for all three function +** SQL function or aggregate, pass NULL pointers for all three function ** callbacks. ** -** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, ** then it is destructor for the application data pointer. ** The destructor is invoked when the function is deleted, either by being ** overloaded or when the database connection closes.)^ @@ -4054,7 +4299,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** The xFunc (for scalar functions) or xStep (for aggregates) parameters ** to [sqlite3_create_function()] and [sqlite3_create_function16()] ** define callbacks that implement the SQL functions and aggregates. -** The 4th parameter to these callbacks is an array of pointers to +** The 3rd parameter to these callbacks is an array of pointers to ** [protected sqlite3_value] objects. There is one [sqlite3_value] object for ** each parameter to the SQL function. These routines are used to ** extract values from the [sqlite3_value] objects. @@ -4381,7 +4626,7 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); ** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin ** on an even byte address. ** -** ^The fourth argument, pArg, is a application data pointer that is passed +** ^The fourth argument, pArg, is an application data pointer that is passed ** through as the first argument to the collating function callback. ** ** ^The fifth argument, xCallback, is a pointer to the collating function. @@ -4397,7 +4642,7 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); ** by the eTextRep argument. The collating function must return an ** integer that is negative, zero, or positive ** if the first string is less than, equal to, or greater than the second, -** respectively. A collating function must alway return the same answer +** respectively. A collating function must always return the same answer ** given the same inputs. If two or more collating functions are registered ** to the same collation name (using different eTextRep values) then all ** must give an equivalent answer when invoked with equivalent strings. @@ -4809,7 +5054,7 @@ SQLITE_API int sqlite3_release_memory(int); **
  • Memory accounting is disabled using a combination of the ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. -**
  • An alternative page cache implementation is specifed using +**
  • An alternative page cache implementation is specified using ** [sqlite3_config]([SQLITE_CONFIG_PCACHE],...). **
  • The page cache allocates from its own memory pool supplied ** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than @@ -5030,7 +5275,7 @@ typedef struct sqlite3_module sqlite3_module; ** CAPI3REF: Virtual Table Object ** KEYWORDS: sqlite3_module {virtual table module} ** -** This structure, sometimes called a a "virtual table module", +** This structure, sometimes called a "virtual table module", ** defines the implementation of a [virtual tables]. ** This structure consists mostly of methods for the module. ** @@ -5070,6 +5315,11 @@ struct sqlite3_module { void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg); int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); }; /* @@ -5342,7 +5592,7 @@ typedef struct sqlite3_blob sqlite3_blob; ** This is true if any column of the row is changed, even a column ** other than the one the BLOB handle is open on.)^ ** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for -** a expired BLOB handle fail with an return code of [SQLITE_ABORT]. +** an expired BLOB handle fail with a return code of [SQLITE_ABORT]. ** ^(Changes written into a BLOB prior to the BLOB expiring are not ** rolled back by the expiration of the BLOB. Such changes will eventually ** commit if the transaction continues to completion.)^ @@ -5752,7 +6002,7 @@ struct sqlite3_mutex_methods { ** ** ^If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since -** clearly the mutex cannot be held if it does not exist. But the +** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is @@ -5782,7 +6032,8 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ #define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */ #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ -#define SQLITE_MUTEX_STATIC_LRU2 7 /* lru page list */ +#define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ /* ** CAPI3REF: Retrieve the mutex for a database connection @@ -5874,7 +6125,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 #define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LAST 18 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 +#define SQLITE_TESTCTRL_LAST 19 /* ** CAPI3REF: SQLite Runtime Status @@ -5883,7 +6135,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes -** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ +** are of the form [status parameters | SQLITE_STATUS_...].)^ ** ^The current value of the parameter is returned into *pCurrent. ** ^The highest recorded value is returned in *pHighwater. ^If the ** resetFlag is true, then the highest record value is reset after @@ -5910,12 +6162,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF /* ** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} ** ** These integer constants designate various run-time status parameters ** that can be returned by [sqlite3_status()]. ** **
    -** ^(
    SQLITE_STATUS_MEMORY_USED
    +** [[SQLITE_STATUS_MEMORY_USED]] ^(
    SQLITE_STATUS_MEMORY_USED
    **
    This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application @@ -5925,22 +6178,24 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** this parameter. The amount returned is the sum of the allocation ** sizes as reported by the xSize method in [sqlite3_mem_methods].
    )^ ** -** ^(
    SQLITE_STATUS_MALLOC_SIZE
    +** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
    SQLITE_STATUS_MALLOC_SIZE
    **
    This parameter records the largest memory allocation request ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
    )^ ** -** ^(
    SQLITE_STATUS_MALLOC_COUNT
    -**
    This parameter records the number of separate memory allocations.
    )^ +** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations +** currently checked out.
    )^ ** -** ^(
    SQLITE_STATUS_PAGECACHE_USED
    +** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
    SQLITE_STATUS_PAGECACHE_USED
    **
    This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
    )^ ** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] @@ -5950,13 +6205,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.
    )^ ** -** ^(
    SQLITE_STATUS_PAGECACHE_SIZE
    +** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
    SQLITE_STATUS_PAGECACHE_SIZE
    **
    This parameter records the largest memory allocation request ** handed to [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
    )^ ** -** ^(
    SQLITE_STATUS_SCRATCH_USED
    +** [[SQLITE_STATUS_SCRATCH_USED]] ^(
    SQLITE_STATUS_SCRATCH_USED
    **
    This parameter returns the number of allocations used out of the ** [scratch memory allocator] configured using ** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not @@ -5964,7 +6219,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** outstanding at time, this parameter also reports the number of threads ** using scratch memory at the same time.
    )^ ** -** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    +** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory ** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values @@ -5974,13 +6229,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** slots were available. **
    )^ ** -** ^(
    SQLITE_STATUS_SCRATCH_SIZE
    +** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
    SQLITE_STATUS_SCRATCH_SIZE
    **
    This parameter records the largest memory allocation request ** handed to [scratch memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
    )^ ** -** ^(
    SQLITE_STATUS_PARSER_STACK
    +** [[SQLITE_STATUS_PARSER_STACK]] ^(
    SQLITE_STATUS_PARSER_STACK
    **
    This parameter records the deepest parser stack. It is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    )^ **
    @@ -6005,9 +6260,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that +** [SQLITE_DBSTATUS options], that ** determines the parameter to interrogate. The set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely +** [SQLITE_DBSTATUS options] is likely ** to grow in future releases of SQLite. ** ** ^The current value of the requested parameter is written into *pCur @@ -6024,6 +6279,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r /* ** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} ** ** These constants are the available integer "verbs" that can be passed as ** the second argument to the [sqlite3_db_status()] interface. @@ -6035,16 +6291,37 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** if a discontinued or unsupported verb is invoked. ** **
    -** ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    +** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    **
    This parameter returns the number of lookaside memory slots currently ** checked out.
    )^ ** -** ^(
    SQLITE_DBSTATUS_CACHE_USED
    +** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_HIT
    +**
    This parameter returns the number malloc attempts that were +** satisfied using lookaside memory. Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
    +**
    This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to the amount of +** memory requested being larger than the lookaside slot size. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
    +**
    This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to all lookaside +** memory already being in use. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    **
    This parameter returns the approximate number of of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** -** ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    **
    This parameter returns the approximate number of of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ @@ -6053,7 +6330,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** -** ^(
    SQLITE_DBSTATUS_STMT_USED
    +** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    **
    This parameter returns the approximate number of of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ @@ -6061,18 +6338,21 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r **
    **
    */ -#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 -#define SQLITE_DBSTATUS_CACHE_USED 1 -#define SQLITE_DBSTATUS_SCHEMA_USED 2 -#define SQLITE_DBSTATUS_STMT_USED 3 -#define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 +#define SQLITE_DBSTATUS_CACHE_USED 1 +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 +#define SQLITE_DBSTATUS_MAX 6 /* Largest defined DBSTATUS */ /* ** CAPI3REF: Prepared Statement Status ** ** ^(Each prepared statement maintains various -** [SQLITE_STMTSTATUS_SORT | counters] that measure the number +** [SQLITE_STMTSTATUS counters] that measure the number ** of times it has performed specific operations.)^ These counters can ** be used to monitor the performance characteristics of the prepared ** statements. For example, if the number of table steps greatly exceeds @@ -6083,7 +6363,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** ^(This interface is used to retrieve and reset counter values from ** a [prepared statement]. The first argument is the prepared statement ** object to be interrogated. The second argument -** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter] +** is an integer code for a specific [SQLITE_STMTSTATUS counter] ** to be interrogated.)^ ** ^The current value of the requested counter is returned. ** ^If the resetFlg is true, then the counter is reset to zero after this @@ -6095,24 +6375,25 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} ** ** These preprocessor macros define integer codes that name counter ** values associated with the [sqlite3_stmt_status()] interface. ** The meanings of the various counters are as follows: ** **
    -**
    SQLITE_STMTSTATUS_FULLSCAN_STEP
    +** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
    SQLITE_STMTSTATUS_FULLSCAN_STEP
    **
    ^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter ** may indicate opportunities for performance improvement through ** careful use of indices.
    ** -**
    SQLITE_STMTSTATUS_SORT
    +** [[SQLITE_STMTSTATUS_SORT]]
    SQLITE_STMTSTATUS_SORT
    **
    ^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to ** improvement performance through careful use of indices.
    ** -**
    SQLITE_STMTSTATUS_AUTOINDEX
    +** [[SQLITE_STMTSTATUS_AUTOINDEX]]
    SQLITE_STMTSTATUS_AUTOINDEX
    **
    ^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to @@ -6163,6 +6444,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** +** [[the xInit() page cache method]] ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() @@ -6173,6 +6455,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** built-in default page cache is used instead of the application defined ** page cache.)^ ** +** [[the xShutdown() page cache method]] ** ^The xShutdown() method is called by [sqlite3_shutdown()]. ** It can be used to clean up ** any outstanding resources before process shutdown, if required. @@ -6187,17 +6470,20 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** +** [[the xCreate() page cache methods]] ** ^SQLite invokes the xCreate() method to construct a new cache instance. ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. SQLite will use the +** increment (here called "R") of less than 250. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. -** ^R is constant for a particular build of SQLite. ^The second argument to +** ^(R is constant for a particular build of SQLite. Except, there are two +** distinct values of R when SQLite is compiled with the proprietary +** ZIPVFS extension.)^ ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation @@ -6209,6 +6495,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** +** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using @@ -6216,20 +6503,22 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** +** [[the xPagecount() page cache methods]] ** The xPagecount() method must return the number of pages currently ** stored in the cache, both pinned and unpinned. ** +** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to ** the page, or a NULL pointer. ** A "page", in this context, means a buffer of szPage bytes aligned at an ** 8-byte boundary. The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** minimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** ** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content ** intact. If the requested page is not already in the cache, then the -** behavior of the cache implementation should use the value of the createFlag +** cache implementation should use the value of the createFlag ** parameter to help it determined what action to take: ** ** @@ -6247,6 +6536,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** attempt to unpin one or more cache pages by spilling the content of ** pinned pages to disk and synching the operating system disk cache. ** +** [[the xUnpin() page cache method]] ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. @@ -6259,6 +6549,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** call to xUnpin() unpins the page regardless of the number of prior calls ** to xFetch(). ** +** [[the xRekey() page cache methods]] ** The xRekey() method is used to change the key value associated with the ** page passed as the second argument. If the cache ** previously contains an entry associated with newKey, it must be @@ -6271,6 +6562,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** +** [[the xDestroy() page cache method]] ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] @@ -6313,11 +6605,12 @@ typedef struct sqlite3_backup sqlite3_backup; ** ** See Also: [Using the SQLite Online Backup API] ** -** ^Exclusive access is required to the destination database for the -** duration of the operation. ^However the source database is only -** read-locked while it is actually being read; it is not locked -** continuously for the entire backup operation. ^Thus, the backup may be -** performed on a live source database without preventing other users from +** ^SQLite holds a write transaction open on the destination database file +** for the duration of the backup operation. +** ^The source database is read-locked only while it is being read; +** it is not locked continuously for the entire backup operation. +** ^Thus, the backup may be performed on a live source database without +** preventing other database connections from ** reading or writing to the source database while the backup is underway. ** ** ^(To perform a backup operation: @@ -6332,7 +6625,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** There should be exactly one call to sqlite3_backup_finish() for each ** successful call to sqlite3_backup_init(). ** -** sqlite3_backup_init() +** [[sqlite3_backup_init()]] sqlite3_backup_init() ** ** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the ** [database connection] associated with the destination database @@ -6344,11 +6637,11 @@ typedef struct sqlite3_backup sqlite3_backup; ** sqlite3_backup_init(D,N,S,M) identify the [database connection] ** and database name of the source database, respectively. ** ^The source and destination [database connections] (parameters S and D) -** must be different or else sqlite3_backup_init(D,N,S,M) will file with +** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is -** returned and an error code and error message are store3d in the +** returned and an error code and error message are stored in the ** destination [database connection] D. ** ^The error code and message for the failed call to sqlite3_backup_init() ** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or @@ -6359,13 +6652,13 @@ typedef struct sqlite3_backup sqlite3_backup; ** sqlite3_backup_finish() functions to perform the specified backup ** operation. ** -** sqlite3_backup_step() +** [[sqlite3_backup_step()]] sqlite3_backup_step() ** ** ^Function sqlite3_backup_step(B,N) will copy up to N pages between ** the source and destination databases specified by [sqlite3_backup] object B. ** ^If N is negative, all remaining source pages are copied. ** ^If sqlite3_backup_step(B,N) successfully copies N pages and there -** are still more pages to be copied, then the function resturns [SQLITE_OK]. +** are still more pages to be copied, then the function returns [SQLITE_OK]. ** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages ** from source to destination, then it returns [SQLITE_DONE]. ** ^If an error occurs while running sqlite3_backup_step(B,N), @@ -6379,7 +6672,7 @@ typedef struct sqlite3_backup sqlite3_backup; **
  • the destination database was opened read-only, or **
  • the destination database is using write-ahead-log journaling ** and the destination and source page sizes differ, or -**
  • The destination database is an in-memory database and the +**
  • the destination database is an in-memory database and the ** destination and source page sizes differ. ** )^ ** @@ -6416,7 +6709,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** by the backup operation, then the backup database is automatically ** updated at the same time. ** -** sqlite3_backup_finish() +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() ** ** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the ** application wishes to abandon the backup operation, the application @@ -6439,7 +6732,8 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** sqlite3_backup_remaining(), sqlite3_backup_pagecount() +** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed @@ -6710,7 +7004,8 @@ SQLITE_API void *sqlite3_wal_hook( ** from SQL. ** ** ^Every new [database connection] defaults to having the auto-checkpoint -** enabled with a threshold of 1000 pages. The use of this interface +** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] +** pages. The use of this interface ** is only necessary if the default setting is found to be suboptimal ** for a particular application. */ @@ -6729,9 +7024,189 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the ** [wal_autocheckpoint pragma] can be used to cause this interface to be ** run whenever the WAL reaches a certain size threshold. +** +** See also: [sqlite3_wal_checkpoint_v2()] */ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); +/* +** CAPI3REF: Checkpoint a database +** +** Run a checkpoint operation on WAL database zDb attached to database +** handle db. The specific operation is determined by the value of the +** eMode parameter: +** +**
    +**
    SQLITE_CHECKPOINT_PASSIVE
    +** Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish. Sync the db file if all frames in the log +** are checkpointed. This mode is the same as calling +** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** +**
    SQLITE_CHECKPOINT_FULL
    +** This mode blocks (calls the busy-handler callback) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. It then checkpoints all frames in the log file and syncs the +** database file. This call blocks database writers while it is running, +** but not database readers. +** +**
    SQLITE_CHECKPOINT_RESTART
    +** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after +** checkpointing the log file it blocks (calls the busy-handler callback) +** until all readers are reading from the database file only. This ensures +** that the next client to write to the database file restarts the log file +** from the beginning. This call blocks database writers while it is running, +** but not database readers. +**
    +** +** If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to +** the total number of checkpointed frames (including any that were already +** checkpointed when this function is called). *pnLog and *pnCkpt may be +** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. +** If no values are available because of an error, they are both set to -1 +** before returning to communicate this to the caller. +** +** All calls obtain an exclusive "checkpoint" lock on the database file. If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive +** "writer" lock on the database file. If the writer lock cannot be obtained +** immediately, and a busy-handler is configured, it is invoked and the writer +** lock retried until either the busy-handler returns 0 or the lock is +** successfully obtained. The busy-handler is also invoked while waiting for +** database readers as described above. If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. SQLITE_BUSY is returned in this case. +** +** If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned to the caller. If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code returned to the caller immediately. If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); + +/* +** CAPI3REF: Checkpoint operation parameters +** +** These constants can be used as the 3rd parameter to +** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] +** documentation for additional information about the meaning and use of +** each of these values. +*/ +#define SQLITE_CHECKPOINT_PASSIVE 0 +#define SQLITE_CHECKPOINT_FULL 1 +#define SQLITE_CHECKPOINT_RESTART 2 + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** At present, there is only one option that may be configured using +** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +** may be added in the future. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
    +**
    SQLITE_VTAB_CONSTRAINT_SUPPORT +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
    +*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Conflict resolution modes +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + + + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. @@ -7397,6 +7872,7 @@ typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; +typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WherePlan WherePlan; typedef struct WhereInfo WhereInfo; @@ -7450,21 +7926,10 @@ typedef struct WhereLevel WhereLevel; typedef struct Btree Btree; typedef struct BtCursor BtCursor; typedef struct BtShared BtShared; -typedef struct BtreeMutexArray BtreeMutexArray; - -/* -** This structure records all of the Btrees that need to hold -** a mutex before we enter sqlite3VdbeExec(). The Btrees are -** are placed in aBtree[] in order of aBtree[]->pBt. That way, -** we can always lock and unlock them all quickly. -*/ -struct BtreeMutexArray { - int nMutex; - Btree *aBtree[SQLITE_MAX_ATTACHED+1]; -}; SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ const char *zFilename, /* Name of database file to open */ sqlite3 *db, /* Associated database connection */ Btree **ppBtree, /* Return open Btree* here */ @@ -7498,7 +7963,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); -SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*); SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); @@ -7618,7 +8083,7 @@ SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); #endif #ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*); +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); #endif /* @@ -7635,30 +8100,28 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); -SQLITE_PRIVATE void sqlite3BtreeMutexArrayEnter(BtreeMutexArray*); -SQLITE_PRIVATE void sqlite3BtreeMutexArrayLeave(BtreeMutexArray*); -SQLITE_PRIVATE void sqlite3BtreeMutexArrayInsert(BtreeMutexArray*, Btree*); #ifndef NDEBUG /* These routines are used inside assert() statements only. */ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree*); SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3*); +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); #endif #else +# define sqlite3BtreeSharable(X) 0 # define sqlite3BtreeLeave(X) # define sqlite3BtreeEnterCursor(X) # define sqlite3BtreeLeaveCursor(X) # define sqlite3BtreeLeaveAll(X) -# define sqlite3BtreeMutexArrayEnter(X) -# define sqlite3BtreeMutexArrayLeave(X) -# define sqlite3BtreeMutexArrayInsert(X,Y) # define sqlite3BtreeHoldsMutex(X) 1 # define sqlite3BtreeHoldsAllMutexes(X) 1 +# define sqlite3SchemaMutexHeld(X,Y,Z) 1 #endif @@ -7777,7 +8240,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ #define P4_VDBEFUNC (-7) /* P4 is a pointer to a VdbeFunc structure */ #define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ -#define P4_TRANSIENT (-9) /* P4 is a pointer to a transient string */ +#define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ #define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ #define P4_MPRINTF (-11) /* P4 is a string obtained from sqlite3_mprintf() */ #define P4_REAL (-12) /* P4 is a 64-bit floating point value */ @@ -8027,6 +8490,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); @@ -8040,7 +8504,7 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); +SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); @@ -8049,6 +8513,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); SQLITE_PRIVATE void sqlite3VdbeTrace(Vdbe*,FILE*); #endif SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); @@ -8228,7 +8693,7 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager); -SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); @@ -8812,9 +9277,24 @@ struct Db { /* ** An instance of the following structure stores a database schema. +** +** Most Schema objects are associated with a Btree. The exception is +** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. +** In shared cache mode, a single Schema object can be shared by multiple +** Btrees that refer to the same underlying BtShared object. +** +** Schema objects are automatically deallocated when the last Btree that +** references them is destroyed. The TEMP Schema is manually freed by +** sqlite3_close(). +* +** A thread must be holding a mutex on the corresponding Btree in order +** to access Schema content. This implies that the thread must also be +** holding a mutex on the sqlite3 connection pointer that owns the Btree. +** For a TEMP Schema, only the connection mutex is required. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ + int iGeneration; /* Generation counter. Incremented with each change */ Hash tblHash; /* All tables indexed by name */ Hash idxHash; /* All (named) indices indexed by name */ Hash trigHash; /* All triggers indexed by name */ @@ -8881,6 +9361,7 @@ struct Lookaside { u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ int nOut; /* Number of buffers currently checked out */ int mxOut; /* Highwater mark for nOut */ + int anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ LookasideSlot *pFree; /* List of available buffers */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ @@ -8930,7 +9411,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ Db *aDb; /* All backends */ int flags; /* Miscellaneous flags. See below */ - int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ + unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ u8 autoCommit; /* The auto-commit flag. */ @@ -8939,6 +9420,7 @@ struct sqlite3 { u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ + u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ @@ -8959,6 +9441,7 @@ struct sqlite3 { struct Vdbe *pVdbe; /* List of active virtual machines */ int activeVdbeCnt; /* Number of VDBEs currently executing */ int writeVdbeCnt; /* Number of active VDBEs that are writing */ + int vdbeExecCnt; /* Number of nested calls to VdbeExec() */ void (*xTrace)(void*,const char*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ void (*xProfile)(void*,const char*,u64); /* Profiling function */ @@ -8996,7 +9479,7 @@ struct sqlite3 { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE Hash aModule; /* populated by sqlite3_create_module() */ - Table *pVTab; /* vtab with active Connect/Create method */ + VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ int nVTrans; /* Allocated size of aVTrans */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ @@ -9066,6 +9549,7 @@ struct sqlite3 { #define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ #define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ +#define SQLITE_EnableTrigger 0x40000000 /* True to enable triggers */ /* ** Bits of the sqlite3.flags field that are used by the @@ -9079,6 +9563,7 @@ struct sqlite3 { #define SQLITE_IndexCover 0x10 /* Disable index covering table */ #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ +#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* @@ -9324,7 +9809,7 @@ struct CollSeq { ** schema is shared, as the implementation often stores the database ** connection handle passed to it via the xConnect() or xCreate() method ** during initialization internally. This database connection handle may -** then used by the virtual table implementation to access real tables +** then be used by the virtual table implementation to access real tables ** within the database. So that they appear as part of the callers ** transaction, these accesses need to be made via the same database ** connection as that used to execute SQL operations on the virtual table. @@ -9358,6 +9843,8 @@ struct VTable { Module *pMod; /* Pointer to module implementation */ sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ + int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ }; @@ -9602,6 +10089,7 @@ struct Index { int tnum; /* Page containing root of this index in database file */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ u8 autoIndex; /* True if is automatically created (ex: by UNIQUE) */ + u8 bUnordered; /* Use this index for == or IN queries only */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ Schema *pSchema; /* Schema containing this index */ @@ -9765,7 +10253,7 @@ struct Expr { u16 flags; /* Various flags. EP_* See below */ union { char *zToken; /* Token value. Zero terminated and dequoted */ - int iValue; /* Integer value if EP_IntValue */ + int iValue; /* Non-negative integer value if EP_IntValue */ } u; /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no @@ -10265,6 +10753,15 @@ struct TriggerPrg { TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */ }; +/* +** The yDbMask datatype for the bitmask of all attached databases. +*/ +#if SQLITE_MAX_ATTACHED>30 + typedef sqlite3_uint64 yDbMask; +#else + typedef unsigned int yDbMask; +#endif + /* ** An SQL parser context. A copy of this structure is passed through ** the parser and down into all the parser action routine in order to @@ -10313,8 +10810,8 @@ struct Parse { int iReg; /* Reg with value of this column. 0 means none. */ int lru; /* Least recently used entry has the smallest value */ } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ - u32 writeMask; /* Start a write transaction on these databases */ - u32 cookieMask; /* Bitmask of schema verified databases */ + yDbMask writeMask; /* Start a write transaction on these databases */ + yDbMask cookieMask; /* Bitmask of schema verified databases */ u8 isMultiWrite; /* True if statement may affect/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ int cookieGoto; /* Address of OP_Goto to cookie verifier subroutine */ @@ -10342,9 +10839,8 @@ struct Parse { ** each recursion */ int nVar; /* Number of '?' variables seen in the SQL so far */ - int nVarExpr; /* Number of used slots in apVarExpr[] */ - int nVarExprAlloc; /* Number of allocated slots in apVarExpr[] */ - Expr **apVarExpr; /* Pointers to :aaa and $aaaa wildcard expressions */ + int nzVar; /* Number of available slots in azVar[] */ + char **azVar; /* Pointers to names of parameters */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ int nAlias; /* Number of aliased result set columns */ int nAliasAlloc; /* Number of allocated slots for aAlias[] */ @@ -10536,6 +11032,7 @@ struct Sqlite3Config { int bMemstat; /* True to enable memory status */ int bCoreMutex; /* True to enable core mutexing */ int bFullMutex; /* True to enable full mutexing */ + int bOpenUri; /* True to interpret filenames as URIs */ int mxStrlen; /* Maximum string length */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ @@ -10564,6 +11061,7 @@ struct Sqlite3Config { int nRefInitMutex; /* Number of users of pInitMutex */ void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ + int bLocaltimeFault; /* True to fail localtime() calls */ }; /* @@ -10785,6 +11283,8 @@ SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*); SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,Select*); +SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); @@ -10884,6 +11384,7 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void); SQLITE_PRIVATE void sqlite3PrngResetState(void); SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*); SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); +SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); SQLITE_PRIVATE void sqlite3BeginTransaction(Parse*, int); SQLITE_PRIVATE void sqlite3CommitTransaction(Parse*); SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse*); @@ -10988,7 +11489,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3Atoi(const char*); SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); -SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); +SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8*, const u8**); /* ** Routines to read and write variable-length integers. These used to @@ -11034,6 +11535,7 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); @@ -11045,6 +11547,16 @@ SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); +SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3MulInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3AbsInt32(int); +#ifdef SQLITE_ENABLE_8_3_NAMES +SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); +#else +# define sqlite3FileSuffix3(X,Y) +#endif +SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); @@ -11069,7 +11581,7 @@ SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; SQLITE_PRIVATE int sqlite3PendingByte; #endif #endif -SQLITE_PRIVATE void sqlite3RootPageMoved(Db*, int, int); +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, int, int); SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); SQLITE_PRIVATE void sqlite3AlterFunctions(void); SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); @@ -11096,7 +11608,7 @@ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse*, int, int); -SQLITE_PRIVATE void sqlite3SchemaFree(void *); +SQLITE_PRIVATE void sqlite3SchemaClear(void *); SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *, Index *); @@ -11154,6 +11666,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) +# define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK #else SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **); @@ -11162,6 +11675,7 @@ SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); @@ -11183,7 +11697,7 @@ SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); SQLITE_PRIVATE const char *sqlite3JournalModename(int); -SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3*, int); +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); SQLITE_PRIVATE int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); /* Declarations for functions in fkey.c. All of these are replaced by @@ -11468,7 +11982,9 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { }; #endif - +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 0 +#endif /* ** The following singleton contains the global configuration for @@ -11478,6 +11994,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ 1, /* bCoreMutex */ SQLITE_THREADSAFE==1, /* bFullMutex */ + SQLITE_USE_URI, /* bOpenUri */ 0x7ffffffe, /* mxStrlen */ 100, /* szLookaside */ 500, /* nLookaside */ @@ -11505,6 +12022,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nRefInitMutex */ 0, /* xLog */ 0, /* pLogArg */ + 0, /* bLocaltimeFault */ }; @@ -11731,6 +12249,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_AUTOMATIC_INDEX "OMIT_AUTOMATIC_INDEX", #endif +#ifdef SQLITE_OMIT_AUTORESET + "OMIT_AUTORESET", +#endif #ifdef SQLITE_OMIT_AUTOVACUUM "OMIT_AUTOVACUUM", #endif @@ -12006,16 +12527,14 @@ typedef unsigned char Bool; ** ** Every cursor that the virtual machine has open is represented by an ** instance of the following structure. -** -** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is -** really a single row that represents the NEW or OLD pseudo-table of -** a row trigger. The data for the row is stored in VdbeCursor.pData and -** the rowid is in VdbeCursor.iKey. */ struct VdbeCursor { BtCursor *pCursor; /* The cursor structure of the backend */ + Btree *pBt; /* Separate file holding temporary table */ + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ int iDb; /* Index of cursor database in db->aDb[] (or -1) */ - i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ + int pseudoTableReg; /* Register holding pseudotable content. */ + int nField; /* Number of fields in the header */ Bool zeroed; /* True if zeroed out and ready for reuse */ Bool rowidIsValid; /* True if lastRowid is valid */ Bool atFirst; /* True if pointing to first entry */ @@ -12025,14 +12544,11 @@ struct VdbeCursor { Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ - i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - Btree *pBt; /* Separate file holding temporary table */ - int pseudoTableReg; /* Register holding pseudotable content. */ - KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ - int nField; /* Number of fields in the header */ - i64 seqCount; /* Sequence counter */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ + i64 seqCount; /* Sequence counter */ + i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ + i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ @@ -12104,25 +12620,19 @@ struct VdbeFrame { /* ** Internally, the vdbe manipulates nearly all SQL values as Mem ** structures. Each Mem struct may cache multiple representations (string, -** integer etc.) of the same value. A value (and therefore Mem structure) -** has the following properties: -** -** Each value has a manifest type. The manifest type of the value stored -** in a Mem struct is returned by the MemType(Mem*) macro. The type is -** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or -** SQLITE_BLOB. +** integer etc.) of the same value. */ struct Mem { + sqlite3 *db; /* The associated database connection */ + char *z; /* String or BLOB value */ + double r; /* Real value */ union { - i64 i; /* Integer value. */ + i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; - double r; /* Real value */ - sqlite3 *db; /* The associated database connection */ - char *z; /* String or BLOB value */ int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ @@ -12146,9 +12656,6 @@ struct Mem { ** database (see below for exceptions). If the MEM_Term flag is also ** set, then the string is nul terminated. The MEM_Int and MEM_Real ** flags may coexist with the MEM_Str flag. -** -** Multiple of these values can appear in Mem.flags. But only one -** at a time can appear in Mem.type. */ #define MEM_Null 0x0001 /* Value is NULL */ #define MEM_Str 0x0002 /* Value is a string */ @@ -12231,23 +12738,11 @@ struct sqlite3_context { CollSeq *pColl; /* Collating sequence */ }; -/* -** A Set structure is used for quick testing to see if a value -** is part of a small set. Sets are used to implement code like -** this: -** x.y IN ('hi','hoo','hum') -*/ -typedef struct Set Set; -struct Set { - Hash hash; /* A set is just a hash table */ - HashElem *prev; /* Previously accessed hash elemen */ -}; - /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. ** -** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile() +** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare() ** is really a pointer to an instance of this structure. ** ** The Vdbe.inVtabMethod variable is set to non-zero for the duration of @@ -12260,31 +12755,31 @@ struct Set { */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + Op *aOp; /* Space to hold the virtual machine's program */ + Mem *aMem; /* The memory locations */ + Mem **apArg; /* Arguments to currently executing user function */ + Mem *aColName; /* Column names to return */ + Mem *pResultSet; /* Pointer to an array of results */ + int nMem; /* Number of memory locations currently allocated */ int nOp; /* Number of instructions in the program */ int nOpAlloc; /* Number of slots allocated for aOp[] */ - Op *aOp; /* Space to hold the virtual machine's program */ int nLabel; /* Number of labels used */ int nLabelAlloc; /* Number of slots allocated in aLabel[] */ int *aLabel; /* Space to hold the labels */ - Mem **apArg; /* Arguments to currently executing user function */ - Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ u16 nResColumn; /* Number of columns in one row of the result set */ u16 nCursor; /* Number of slots in apCsr[] */ + u32 magic; /* Magic number for sanity checking */ + char *zErrMsg; /* Error message written here */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ VdbeCursor **apCsr; /* One element of this array for each open cursor */ - u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ - ynVar nVar; /* Number of entries in aVar[] */ Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ - u32 magic; /* Magic number for sanity checking */ - int nMem; /* Number of memory locations currently allocated */ - Mem *aMem; /* The memory locations */ + ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ - char *zErrMsg; /* Error message written here */ + u8 errorAction; /* Recovery action to do in case of an error */ u8 explain; /* True if EXPLAIN present on SQL command */ u8 changeCntOn; /* True to update the change-counter */ u8 expired; /* True if the VM needs to be recompiled */ @@ -12295,15 +12790,17 @@ struct Vdbe { u8 readOnly; /* True for read-only statements */ u8 isPrepareV2; /* True if prepared with prepare_v2() */ int nChange; /* Number of db changes made since last reset */ - int btreeMask; /* Bitmask of db->aDb[] entries referenced */ - i64 startTime; /* Time when query started - used for profiling */ - BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */ + yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ + yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + int iStatement; /* Statement number (or 0 if has not opened stmt) */ int aCounter[3]; /* Counters used by sqlite3_stmt_status() */ - char *zSql; /* Text of the SQL statement that generated this */ - void *pFree; /* Free this when deleting the vdbe */ +#ifndef SQLITE_OMIT_TRACE + i64 startTime; /* Time when query started - used for profiling */ +#endif i64 nFkConstraint; /* Number of imm. FK constraints this VM */ i64 nStmtDefCons; /* Number of def. constraints when stmt started */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ + char *zSql; /* Text of the SQL statement that generated this */ + void *pFree; /* Free this when deleting the vdbe */ #ifdef SQLITE_DEBUG FILE *trace; /* Write an execution trace here, if not NULL */ #endif @@ -12379,6 +12876,14 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); +#else +# define sqlite3VdbeEnter(X) +# define sqlite3VdbeLeave(X) +#endif + #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe*,Mem*); #endif @@ -12389,12 +12894,6 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); # define sqlite3VdbeCheckFk(p,i) 0 #endif -#ifndef SQLITE_OMIT_SHARED_CACHE -SQLITE_PRIVATE void sqlite3VdbeMutexArrayEnter(Vdbe *p); -#else -# define sqlite3VdbeMutexArrayEnter(p) -#endif - SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); @@ -12513,6 +13012,22 @@ SQLITE_API int sqlite3_db_status( break; } + case SQLITE_DBSTATUS_LOOKASIDE_HIT: + case SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE: + case SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL: { + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_HIT ); + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE ); + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL ); + assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); + assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); + *pCurrent = 0; + *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT]; + if( resetFlag ){ + db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; + } + break; + } + /* ** Return an approximation for the amount of memory currently used ** by all pagers associated with the given database connection. The @@ -12544,6 +13059,7 @@ SQLITE_API int sqlite3_db_status( int i; /* Used to iterate through schemas */ int nByte = 0; /* Used to accumulate return value */ + sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; @@ -12570,6 +13086,7 @@ SQLITE_API int sqlite3_db_status( } } db->pnBytesFreed = 0; + sqlite3BtreeLeaveAll(db); *pHighwater = 0; *pCurrent = nByte; @@ -12656,22 +13173,6 @@ SQLITE_API int sqlite3_db_status( #ifndef SQLITE_OMIT_DATETIME_FUNCS -/* -** On recent Windows platforms, the localtime_s() function is available -** as part of the "Secure CRT". It is essentially equivalent to -** localtime_r() available under most POSIX platforms, except that the -** order of the parameters is reversed. -** -** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. -** -** If the user has not indicated to use localtime_r() or localtime_s() -** already, check for an MSVC build environment that provides -** localtime_s(). -*/ -#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ - defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) -#define HAVE_LOCALTIME_S 1 -#endif /* ** A structure for holding a single date and time. @@ -13017,15 +13518,83 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ + defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#define HAVE_LOCALTIME_S 1 +#endif + #ifndef SQLITE_OMIT_LOCALTIME /* -** Compute the difference (in milliseconds) -** between localtime and UTC (a.k.a. GMT) -** for the time value p where p is in UTC. +** The following routine implements the rough equivalent of localtime_r() +** using whatever operating-system specific localtime facility that +** is available. This routine returns 0 on success and +** non-zero on any kind of error. +** +** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this +** routine will always fail. */ -static sqlite3_int64 localtimeOffset(DateTime *p){ +static int osLocaltime(time_t *t, struct tm *pTm){ + int rc; +#if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ + && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) + struct tm *pX; + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutex); + pX = localtime(t); +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#endif + if( pX ) *pTm = *pX; + sqlite3_mutex_leave(mutex); + rc = pX==0; +#else +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#endif +#if defined(HAVE_LOCALTIME_R) && HAVE_LOCALTIME_R + rc = localtime_r(t, pTm)==0; +#else + rc = localtime_s(pTm, t); +#endif /* HAVE_LOCALTIME_R */ +#endif /* HAVE_LOCALTIME_R || HAVE_LOCALTIME_S */ + return rc; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** Compute the difference (in milliseconds) between localtime and UTC +** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, +** return this value and set *pRc to SQLITE_OK. +** +** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value +** is undefined in this case. +*/ +static sqlite3_int64 localtimeOffset( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx, /* Write error here if one occurs */ + int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +){ DateTime x, y; time_t t; + struct tm sLocal; + + /* Initialize the contents of sLocal to avoid a compiler warning. */ + memset(&sLocal, 0, sizeof(sLocal)); + x = *p; computeYMD_HMS(&x); if( x.Y<1971 || x.Y>=2038 ){ @@ -13043,47 +13612,23 @@ static sqlite3_int64 localtimeOffset(DateTime *p){ x.validJD = 0; computeJD(&x); t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); -#ifdef HAVE_LOCALTIME_R - { - struct tm sLocal; - localtime_r(&t, &sLocal); - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; - } -#elif defined(HAVE_LOCALTIME_S) && HAVE_LOCALTIME_S - { - struct tm sLocal; - localtime_s(&sLocal, &t); - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; - } -#else - { - struct tm *pTm; - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - pTm = localtime(&t); - y.Y = pTm->tm_year + 1900; - y.M = pTm->tm_mon + 1; - y.D = pTm->tm_mday; - y.h = pTm->tm_hour; - y.m = pTm->tm_min; - y.s = pTm->tm_sec; - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + if( osLocaltime(&t, &sLocal) ){ + sqlite3_result_error(pCtx, "local time unavailable", -1); + *pRc = SQLITE_ERROR; + return 0; } -#endif + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; y.validYMD = 1; y.validHMS = 1; y.validJD = 0; y.validTZ = 0; computeJD(&y); + *pRc = SQLITE_OK; return y.iJD - x.iJD; } #endif /* SQLITE_OMIT_LOCALTIME */ @@ -13107,9 +13652,12 @@ static sqlite3_int64 localtimeOffset(DateTime *p){ ** localtime ** utc ** -** Return 0 on success and 1 if there is any kind of error. +** Return 0 on success and 1 if there is any kind of error. If the error +** is in a system call (i.e. localtime()), then an error message is written +** to context pCtx. If the error is an unrecognized modifier, no error is +** written to pCtx. */ -static int parseModifier(const char *zMod, DateTime *p){ +static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ int rc = 1; int n; double r; @@ -13129,9 +13677,8 @@ static int parseModifier(const char *zMod, DateTime *p){ */ if( strcmp(z, "localtime")==0 ){ computeJD(p); - p->iJD += localtimeOffset(p); + p->iJD += localtimeOffset(p, pCtx, &rc); clearYMD_HMS_TZ(p); - rc = 0; } break; } @@ -13152,11 +13699,12 @@ static int parseModifier(const char *zMod, DateTime *p){ else if( strcmp(z, "utc")==0 ){ sqlite3_int64 c1; computeJD(p); - c1 = localtimeOffset(p); - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p); - rc = 0; + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } } #endif break; @@ -13337,9 +13885,8 @@ static int isDate( } } for(i=1; imem5.szAtom ){ @@ -16530,11 +17078,16 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ struct sqlite3_mutex { HMTX mutex; /* Mutex controlling the lock */ int id; /* Mutex type */ - int nRef; /* Number of references */ - TID owner; /* Thread holding this mutex */ +#ifdef SQLITE_DEBUG + int trace; /* True to trace changes */ +#endif }; -#define OS2_MUTEX_INITIALIZER 0,0,0,0 +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { 0, 0, 0 } +#else +#define SQLITE3_MUTEX_INITIALIZER { 0, 0 } +#endif /* ** Initialize and deinitialize the mutex subsystem. @@ -16550,11 +17103,14 @@ static int os2MutexEnd(void){ return SQLITE_OK; } ** to sqlite3_mutex_alloc() is one of these integer constants: ** **
      -**
    • SQLITE_MUTEX_FAST 0 -**
    • SQLITE_MUTEX_RECURSIVE 1 -**
    • SQLITE_MUTEX_STATIC_MASTER 2 -**
    • SQLITE_MUTEX_STATIC_MEM 3 -**
    • SQLITE_MUTEX_STATIC_PRNG 4 +**
    • SQLITE_MUTEX_FAST +**
    • SQLITE_MUTEX_RECURSIVE +**
    • SQLITE_MUTEX_STATIC_MASTER +**
    • SQLITE_MUTEX_STATIC_MEM +**
    • SQLITE_MUTEX_STATIC_MEM2 +**
    • SQLITE_MUTEX_STATIC_PRNG +**
    • SQLITE_MUTEX_STATIC_LRU +**
    • SQLITE_MUTEX_STATIC_LRU2 **
    ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -16568,7 +17124,7 @@ static int os2MutexEnd(void){ return SQLITE_OK; } ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** The other allowed parameters to sqlite3_mutex_alloc() each return -** a pointer to a static preexisting mutex. Three static mutexes are +** a pointer to a static preexisting mutex. Six static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -16598,13 +17154,13 @@ static sqlite3_mutex *os2MutexAlloc(int iType){ } default: { static volatile int isInit = 0; - static sqlite3_mutex staticMutexes[] = { - { OS2_MUTEX_INITIALIZER, }, - { OS2_MUTEX_INITIALIZER, }, - { OS2_MUTEX_INITIALIZER, }, - { OS2_MUTEX_INITIALIZER, }, - { OS2_MUTEX_INITIALIZER, }, - { OS2_MUTEX_INITIALIZER, }, + static sqlite3_mutex staticMutexes[6] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, }; if ( !isInit ){ APIRET rc; @@ -16650,9 +17206,14 @@ static sqlite3_mutex *os2MutexAlloc(int iType){ ** SQLite is careful to deallocate every mutex that it allocates. */ static void os2MutexFree(sqlite3_mutex *p){ - if( p==0 ) return; - assert( p->nRef==0 ); +#ifdef SQLITE_DEBUG + TID tid; + PID pid; + ULONG ulCount; + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + assert( ulCount==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif DosCloseMutexSem( p->mutex ); sqlite3_free( p ); } @@ -16667,26 +17228,29 @@ static int os2MutexHeld(sqlite3_mutex *p){ PID pid; ULONG ulCount; PTIB ptib; - if( p!=0 ) { - DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); - } else { - DosGetInfoBlocks(&ptib, NULL); - tid = ptib->tib_ptib2->tib2_ultid; - } - return p==0 || (p->nRef!=0 && p->owner==tid); + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + if( ulCount==0 || ( ulCount>1 && p->id!=SQLITE_MUTEX_RECURSIVE ) ) + return 0; + DosGetInfoBlocks(&ptib, NULL); + return tid==ptib->tib_ptib2->tib2_ultid; } static int os2MutexNotheld(sqlite3_mutex *p){ TID tid; PID pid; ULONG ulCount; PTIB ptib; - if( p!= 0 ) { - DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); - } else { - DosGetInfoBlocks(&ptib, NULL); - tid = ptib->tib_ptib2->tib2_ultid; - } - return p==0 || p->nRef==0 || p->owner!=tid; + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + if( ulCount==0 ) + return 1; + DosGetInfoBlocks(&ptib, NULL); + return tid!=ptib->tib_ptib2->tib2_ultid; +} +static void os2MutexTrace(sqlite3_mutex *p, char *pAction){ + TID tid; + PID pid; + ULONG ulCount; + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + printf("%s mutex %p (%d) with nRef=%ld\n", pAction, (void*)p, p->trace, ulCount); } #endif @@ -16702,32 +17266,21 @@ static int os2MutexNotheld(sqlite3_mutex *p){ ** more than once, the behavior is undefined. */ static void os2MutexEnter(sqlite3_mutex *p){ - TID tid; - PID holder1; - ULONG holder2; - if( p==0 ) return; assert( p->id==SQLITE_MUTEX_RECURSIVE || os2MutexNotheld(p) ); DosRequestMutexSem(p->mutex, SEM_INDEFINITE_WAIT); - DosQueryMutexSem(p->mutex, &holder1, &tid, &holder2); - p->owner = tid; - p->nRef++; +#ifdef SQLITE_DEBUG + if( p->trace ) os2MutexTrace(p, "enter"); +#endif } static int os2MutexTry(sqlite3_mutex *p){ - int rc; - TID tid; - PID holder1; - ULONG holder2; - if( p==0 ) return SQLITE_OK; + int rc = SQLITE_BUSY; assert( p->id==SQLITE_MUTEX_RECURSIVE || os2MutexNotheld(p) ); - if( DosRequestMutexSem(p->mutex, SEM_IMMEDIATE_RETURN) == NO_ERROR) { - DosQueryMutexSem(p->mutex, &holder1, &tid, &holder2); - p->owner = tid; - p->nRef++; + if( DosRequestMutexSem(p->mutex, SEM_IMMEDIATE_RETURN) == NO_ERROR ) { rc = SQLITE_OK; - } else { - rc = SQLITE_BUSY; +#ifdef SQLITE_DEBUG + if( p->trace ) os2MutexTrace(p, "try"); +#endif } - return rc; } @@ -16738,16 +17291,11 @@ static int os2MutexTry(sqlite3_mutex *p){ ** is not currently allocated. SQLite will never do either. */ static void os2MutexLeave(sqlite3_mutex *p){ - TID tid; - PID holder1; - ULONG holder2; - if( p==0 ) return; - assert( p->nRef>0 ); - DosQueryMutexSem(p->mutex, &holder1, &tid, &holder2); - assert( p->owner==tid ); - p->nRef--; - assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + assert( os2MutexHeld(p) ); DosReleaseMutexSem(p->mutex); +#ifdef SQLITE_DEBUG + if( p->trace ) os2MutexTrace(p, "leave"); +#endif } SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ @@ -16762,6 +17310,9 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ #ifdef SQLITE_DEBUG os2MutexHeld, os2MutexNotheld +#else + 0, + 0 #endif }; @@ -16871,7 +17422,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_MEM2 **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -17281,7 +17832,7 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_MEM2 **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -17405,7 +17956,7 @@ static int winMutexTry(sqlite3_mutex *p){ #endif #ifdef SQLITE_DEBUG if( rc==SQLITE_OK && p->trace ){ - printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + printf("try mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); } #endif return rc; @@ -17724,7 +18275,7 @@ static int mallocWithAlarm(int n, void **pp){ sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmCallback!=0 ){ int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - if( nUsed+nFull >= mem0.alarmThreshold ){ + if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); }else{ @@ -17862,7 +18413,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ pSlot->pNext = mem0.pScratchFree; mem0.pScratchFree = pSlot; mem0.nScratchFree++; - assert( mem0.nScratchFree<=sqlite3GlobalConfig.nScratch ); + assert( mem0.nScratchFree <= (u32)sqlite3GlobalConfig.nScratch ); sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -17965,7 +18516,7 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ ** Change the size of an existing memory allocation */ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ - int nOld, nNew; + int nOld, nNew, nDiff; void *pNew; if( pOld==0 ){ return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ @@ -17988,9 +18539,10 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); - if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nNew-nOld >= - mem0.alarmThreshold ){ - sqlite3MallocAlarm(nNew-nOld); + nDiff = nNew - nOld; + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); } assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); @@ -18074,14 +18626,20 @@ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){ if( db->mallocFailed ){ return 0; } - if( db->lookaside.bEnabled && n<=db->lookaside.sz - && (pBuf = db->lookaside.pFree)!=0 ){ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; + if( db->lookaside.bEnabled ){ + if( n>db->lookaside.sz ){ + db->lookaside.anStat[1]++; + }else if( (pBuf = db->lookaside.pFree)==0 ){ + db->lookaside.anStat[2]++; + }else{ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + db->lookaside.anStat[0]++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; + } + return (void*)pBuf; } - return (void*)pBuf; } } #else @@ -18630,7 +19188,11 @@ SQLITE_PRIVATE void sqlite3VXPrintf( v = va_arg(ap,int); } if( v<0 ){ - longvalue = -v; + if( v==SMALLEST_INT64 ){ + longvalue = ((u64)1)<<63; + }else{ + longvalue = -v; + } prefix = '-'; }else{ longvalue = v; @@ -18993,6 +19555,7 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ return; } }else{ + char *zOld = (p->zText==p->zBase ? 0 : p->zText); i64 szNew = p->nChar; szNew += N + 1; if( szNew > p->mxAlloc ){ @@ -19003,13 +19566,12 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ p->nAlloc = (int)szNew; } if( p->useMalloc==1 ){ - zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); }else{ - zNew = sqlite3_malloc(p->nAlloc); + zNew = sqlite3_realloc(zOld, p->nAlloc); } if( zNew ){ - memcpy(zNew, p->zText, p->nChar); - sqlite3StrAccumReset(p); + if( zOld==0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; }else{ p->mallocFailed = 1; @@ -19164,21 +19726,28 @@ SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ ** current locale settings. This is important for SQLite because we ** are not able to use a "," as the decimal point in place of "." as ** specified by some locales. +** +** Oops: The first two arguments of sqlite3_snprintf() are backwards +** from the snprintf() standard. Unfortunately, it is too late to change +** this without breaking compatibility, so we just have to live with the +** mistake. +** +** sqlite3_vsnprintf() is the varargs version. */ -SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ - char *z; - va_list ap; +SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; - - if( n<=0 ){ - return zBuf; - } + if( n<=0 ) return zBuf; sqlite3StrAccumInit(&acc, zBuf, n, 0); acc.useMalloc = 0; - va_start(ap,zFormat); sqlite3VXPrintf(&acc, 0, zFormat, ap); + return sqlite3StrAccumFinish(&acc); +} +SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ + char *z; + va_list ap; + va_start(ap,zFormat); + z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); va_end(ap); - z = sqlite3StrAccumFinish(&acc); return z; } @@ -19558,11 +20127,11 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFF800)==0xD800 \ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } -SQLITE_PRIVATE int sqlite3Utf8Read( +SQLITE_PRIVATE u32 sqlite3Utf8Read( const unsigned char *zIn, /* First byte of UTF-8 character */ const unsigned char **pzNext /* Write first byte past UTF-8 char here */ ){ - int c; + unsigned int c; /* Same as READ_UTF8() above but without the zTerm parameter. ** For this routine, we assume the UTF8 string is always zero-terminated. @@ -19805,15 +20374,15 @@ SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){ ** This has the effect of making sure that the string is well-formed ** UTF-8. Miscoded characters are removed. ** -** The translation is done in-place (since it is impossible for the -** correct UTF-8 encoding to be longer than a malformed encoding). +** The translation is done in-place and aborted if the output +** overruns the input. */ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char *zIn){ unsigned char *zOut = zIn; unsigned char *zStart = zIn; u32 c; - while( zIn[0] ){ + while( zIn[0] && zOut<=zIn ){ c = sqlite3Utf8Read(zIn, (const u8**)&zIn); if( c!=0xfffd ){ WRITE_UTF8(zOut, c); @@ -19982,8 +20551,8 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ */ #ifdef SQLITE_COVERAGE_TEST SQLITE_PRIVATE void sqlite3Coverage(int x){ - static int dummy = 0; - dummy += x; + static unsigned dummy = 0; + dummy += (unsigned)x; } #endif @@ -20397,14 +20966,17 @@ static int compare2pow63(const char *zNum, int incr){ /* -** Convert zNum to a 64-bit signed integer and write -** the value of the integer into *pNum. -** If zNum is exactly 9223372036854665808, return 2. -** This is a special case as the context will determine -** if it is too big (used as a negative). -** If zNum is not an integer or is an integer that -** is too large to be expressed with 64 bits, -** then return 1. Otherwise return 0. +** Convert zNum to a 64-bit signed integer. +** +** If the zNum value is representable as a 64-bit twos-complement +** integer, then write that value into *pNum and return 0. +** +** If zNum is exactly 9223372036854665808, return 2. This special +** case is broken out because while 9223372036854665808 cannot be a +** signed 64-bit integer, its negative -9223372036854665808 can be. +** +** If zNum is too big for a 64-bit integer and is not +** 9223372036854665808 then return 1. ** ** length is the number of bytes in the string (bytes, not characters). ** The string is not necessarily zero-terminated. The encoding is @@ -20412,7 +20984,7 @@ static int compare2pow63(const char *zNum, int incr){ */ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ int incr = (enc==SQLITE_UTF8?1:2); - i64 v = 0; + u64 u = 0; int neg = 0; /* assume positive */ int i; int c = 0; @@ -20420,20 +20992,26 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc const char *zEnd = zNum + length; if( enc==SQLITE_UTF16BE ) zNum++; while( zNum=zEnd ) goto do_atoi_calc; - if( *zNum=='-' ){ - neg = 1; - zNum+=incr; - }else if( *zNum=='+' ){ - zNum+=incr; + if( zNum='0' && c<='9'; i+=incr){ - v = v*10 + c - '0'; + u = u*10 + c - '0'; + } + if( u>LARGEST_INT64 ){ + *pNum = SMALLEST_INT64; + }else if( neg ){ + *pNum = -(i64)u; + }else{ + *pNum = (i64)u; } - *pNum = neg ? -v : v; testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); @@ -20443,14 +21021,25 @@ do_atoi_calc: return 1; }else if( i<19*incr ){ /* Less than 19 digits, so we know that it fits in 64 bits */ + assert( u<=LARGEST_INT64 ); return 0; }else{ - /* 19-digit numbers must be no larger than 9223372036854775807 if positive - ** or 9223372036854775808 if negative. Note that 9223372036854665808 - ** is 2^63. Return 1 if to large */ - c=compare2pow63(zNum, incr); - if( c==0 && neg==0 ) return 2; /* too big, exactly 9223372036854665808 */ - return c0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 1; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + assert( (*pNum)==SMALLEST_INT64 ); + return neg ? 0 : 2; + } } } @@ -20919,13 +21508,12 @@ SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* ** Translate a single byte of Hex into an integer. ** This routine only works if h really is a valid hexadecimal ** character: 0..9a..fA..F */ -static u8 hexToInt(int h){ +SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); #ifdef SQLITE_ASCII h += 9*(1&(h>>6)); @@ -20935,7 +21523,6 @@ static u8 hexToInt(int h){ #endif return (u8)(h & 0xf); } -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ #if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* @@ -20952,7 +21539,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ n--; if( zBlob ){ for(i=0; i=0 ){ + testcase( iA>0 && LARGEST_INT64 - iA == iB ); + testcase( iA>0 && LARGEST_INT64 - iA == iB - 1 ); + if( iA>0 && LARGEST_INT64 - iA < iB ) return 1; + *pA += iB; + }else{ + testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 1 ); + testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 2 ); + if( iA<0 && -(iA + LARGEST_INT64) > iB + 1 ) return 1; + *pA += iB; + } + return 0; +} +SQLITE_PRIVATE int sqlite3SubInt64(i64 *pA, i64 iB){ + testcase( iB==SMALLEST_INT64+1 ); + if( iB==SMALLEST_INT64 ){ + testcase( (*pA)==(-1) ); testcase( (*pA)==0 ); + if( (*pA)>=0 ) return 1; + *pA -= iB; + return 0; + }else{ + return sqlite3AddInt64(pA, -iB); + } +} +#define TWOPOWER32 (((i64)1)<<32) +#define TWOPOWER31 (((i64)1)<<31) +SQLITE_PRIVATE int sqlite3MulInt64(i64 *pA, i64 iB){ + i64 iA = *pA; + i64 iA1, iA0, iB1, iB0, r; + + iA1 = iA/TWOPOWER32; + iA0 = iA % TWOPOWER32; + iB1 = iB/TWOPOWER32; + iB0 = iB % TWOPOWER32; + if( iA1*iB1 != 0 ) return 1; + assert( iA1*iB0==0 || iA0*iB1==0 ); + r = iA1*iB0 + iA0*iB1; + testcase( r==(-TWOPOWER31)-1 ); + testcase( r==(-TWOPOWER31) ); + testcase( r==TWOPOWER31 ); + testcase( r==TWOPOWER31-1 ); + if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1; + r *= TWOPOWER32; + if( sqlite3AddInt64(&r, iA0*iB0) ) return 1; + *pA = r; + return 0; +} + +/* +** Compute the absolute value of a 32-bit signed integer, of possible. Or +** if the integer has a value of -2147483648, return +2147483647 +*/ +SQLITE_PRIVATE int sqlite3AbsInt32(int x){ + if( x>=0 ) return x; + if( x==(int)0x80000000 ) return 0x7fffffff; + return -x; +} + +#ifdef SQLITE_ENABLE_8_3_NAMES +/* +** If SQLITE_ENABLE_8_3_NAME is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +*/ +SQLITE_PRIVATE void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ + const char *zOk; + zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); + if( zOk && sqlite3GetBoolean(zOk) ){ + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4); + } +} +#endif + /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ /* @@ -21715,20 +22396,35 @@ SQLITE_API int sqlite3_open_file_count = 0; /************** End of os_common.h *******************************************/ /************** Continuing where we left off in os_os2.c *********************/ +/* Forward references */ +typedef struct os2File os2File; /* The file structure */ +typedef struct os2ShmNode os2ShmNode; /* A shared descritive memory node */ +typedef struct os2ShmLink os2ShmLink; /* A connection to shared-memory */ + /* ** The os2File structure is subclass of sqlite3_file specific for the OS/2 ** protability layer. */ -typedef struct os2File os2File; struct os2File { const sqlite3_io_methods *pMethod; /* Always the first entry */ HFILE h; /* Handle for accessing the file */ - char* pathToDel; /* Name of file to delete on close, NULL if not */ - unsigned char locktype; /* Type of lock currently held on this file */ + int flags; /* Flags provided to os2Open() */ + int locktype; /* Type of lock currently held on this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ + char *zFullPathCp; /* Full path name of this file */ + os2ShmLink *pShmLink; /* Instance of shared memory on this file */ }; #define LOCK_TIMEOUT 10L /* the default locking timeout */ +/* +** Missing from some versions of the OS/2 toolkit - +** used to allocate from high memory if possible +*/ +#ifndef OBJ_ANY +# define OBJ_ANY 0x00000400 +#endif + /***************************************************************************** ** The next group of routines implement the I/O methods specified ** by the sqlite3_io_methods object. @@ -21738,21 +22434,24 @@ struct os2File { ** Close a file. */ static int os2Close( sqlite3_file *id ){ - APIRET rc = NO_ERROR; - os2File *pFile; - if( id && (pFile = (os2File*)id) != 0 ){ - OSTRACE(( "CLOSE %d\n", pFile->h )); - rc = DosClose( pFile->h ); - pFile->locktype = NO_LOCK; - if( pFile->pathToDel != NULL ){ - rc = DosForceDelete( (PSZ)pFile->pathToDel ); - free( pFile->pathToDel ); - pFile->pathToDel = NULL; - } - id = 0; - OpenCounter( -1 ); - } + APIRET rc; + os2File *pFile = (os2File*)id; + + assert( id!=0 ); + OSTRACE(( "CLOSE %d (%s)\n", pFile->h, pFile->zFullPathCp )); + + rc = DosClose( pFile->h ); + + if( pFile->flags & SQLITE_OPEN_DELETEONCLOSE ) + DosForceDelete( (PSZ)pFile->zFullPathCp ); + + free( pFile->zFullPathCp ); + pFile->zFullPathCp = NULL; + pFile->locktype = NO_LOCK; + pFile->h = (HFILE)-1; + pFile->flags = 0; + OpenCounter( -1 ); return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; } @@ -21825,10 +22524,21 @@ static int os2Write( ** Truncate an open file to a specified size */ static int os2Truncate( sqlite3_file *id, i64 nByte ){ - APIRET rc = NO_ERROR; + APIRET rc; os2File *pFile = (os2File*)id; + assert( id!=0 ); OSTRACE(( "TRUNCATE %d %lld\n", pFile->h, nByte )); SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + rc = DosSetFileSize( pFile->h, nByte ); return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR_TRUNCATE; } @@ -22192,8 +22902,22 @@ static int os2FileControl(sqlite3_file *id, int op, void *pArg){ ((os2File*)id)->h, ((os2File*)id)->locktype )); return SQLITE_OK; } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((os2File*)id)->szChunk = *(int*)pArg; + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + sqlite3_int64 sz = *(sqlite3_int64*)pArg; + SimulateIOErrorBenign(1); + os2Truncate(id, sz); + SimulateIOErrorBenign(0); + return SQLITE_OK; + } + case SQLITE_FCNTL_SYNC_OMITTED: { + return SQLITE_OK; + } } - return SQLITE_ERROR; + return SQLITE_NOTFOUND; } /* @@ -22207,6 +22931,7 @@ static int os2FileControl(sqlite3_file *id, int op, void *pArg){ ** same for both. */ static int os2SectorSize(sqlite3_file *id){ + UNUSED_PARAMETER(id); return SQLITE_DEFAULT_SECTOR_SIZE; } @@ -22214,7 +22939,8 @@ static int os2SectorSize(sqlite3_file *id){ ** Return a vector of device characteristics. */ static int os2DeviceCharacteristics(sqlite3_file *id){ - return 0; + UNUSED_PARAMETER(id); + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; } @@ -22301,2616 +23027,2637 @@ char *convertCpPathToUtf8( const char *in ){ return out; } -/* -** This vector defines all the methods that can operate on an -** sqlite3_file for os2. -*/ -static const sqlite3_io_methods os2IoMethod = { - 1, /* iVersion */ - os2Close, - os2Read, - os2Write, - os2Truncate, - os2Sync, - os2FileSize, - os2Lock, - os2Unlock, - os2CheckReservedLock, - os2FileControl, - os2SectorSize, - os2DeviceCharacteristics -}; -/*************************************************************************** -** Here ends the I/O methods that form the sqlite3_io_methods object. -** -** The next block of code implements the VFS methods. -****************************************************************************/ +#ifndef SQLITE_OMIT_WAL /* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at pVfs->mxPathname characters. +** Use main database file for interprocess locking. If un-defined +** a separate file is created for this purpose. The file will be +** used only to set file locks. There will be no data written to it. */ -static int getTempname(int nBuf, char *zBuf ){ - static const unsigned char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - int i, j; - char zTempPathBuf[3]; - PSZ zTempPath = (PSZ)&zTempPathBuf; - if( sqlite3_temp_directory ){ - zTempPath = sqlite3_temp_directory; - }else{ - if( DosScanEnv( (PSZ)"TEMP", &zTempPath ) ){ - if( DosScanEnv( (PSZ)"TMP", &zTempPath ) ){ - if( DosScanEnv( (PSZ)"TMPDIR", &zTempPath ) ){ - ULONG ulDriveNum = 0, ulDriveMap = 0; - DosQueryCurrentDisk( &ulDriveNum, &ulDriveMap ); - sprintf( (char*)zTempPath, "%c:", (char)( 'A' + ulDriveNum - 1 ) ); - } - } - } - } - /* Strip off a trailing slashes or backslashes, otherwise we would get * - * multiple (back)slashes which causes DosOpen() to fail. * - * Trailing spaces are not allowed, either. */ - j = sqlite3Strlen30(zTempPath); - while( j > 0 && ( zTempPath[j-1] == '\\' || zTempPath[j-1] == '/' - || zTempPath[j-1] == ' ' ) ){ - j--; - } - zTempPath[j] = '\0'; - if( !sqlite3_temp_directory ){ - char *zTempPathUTF = convertCpPathToUtf8( zTempPath ); - sqlite3_snprintf( nBuf-30, zBuf, - "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPathUTF ); - free( zTempPathUTF ); - }else{ - sqlite3_snprintf( nBuf-30, zBuf, - "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath ); - } - j = sqlite3Strlen30( zBuf ); - sqlite3_randomness( 20, &zBuf[j] ); - for( i = 0; i < 20; i++, j++ ){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; - } - zBuf[j] = 0; - OSTRACE(( "TEMP FILENAME: %s\n", zBuf )); - return SQLITE_OK; -} +#define SQLITE_OS2_NO_WAL_LOCK_FILE +#if 0 +static void _ERR_TRACE( const char *fmt, ... ) { + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fflush(stderr); +} +#define ERR_TRACE(rc, msg) \ + if( (rc) != SQLITE_OK ) _ERR_TRACE msg; +#else +#define ERR_TRACE(rc, msg) +#endif /* -** Turn a relative pathname into a full pathname. Write the full -** pathname into zFull[]. zFull[] will be at least pVfs->mxPathname -** bytes in size. +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect os2ShmNodeList. +** +** Function os2ShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** os2ShmEnterMutex() +** assert( os2ShmMutexHeld() ); +** os2ShmLeaveMutex() */ -static int os2FullPathname( - sqlite3_vfs *pVfs, /* Pointer to vfs object */ - const char *zRelative, /* Possibly relative input path */ - int nFull, /* Size of output buffer in bytes */ - char *zFull /* Output buffer */ -){ - char *zRelativeCp = convertUtf8PathToCp( zRelative ); - char zFullCp[CCHMAXPATH] = "\0"; - char *zFullUTF; - APIRET rc = DosQueryPathInfo( zRelativeCp, FIL_QUERYFULLNAME, zFullCp, - CCHMAXPATH ); - free( zRelativeCp ); - zFullUTF = convertCpPathToUtf8( zFullCp ); - sqlite3_snprintf( nFull, zFull, zFullUTF ); - free( zFullUTF ); - return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; +static void os2ShmEnterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } - +static void os2ShmLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +#ifdef SQLITE_DEBUG +static int os2ShmMutexHeld(void) { + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +int GetCurrentProcessId(void) { + PPIB pib; + DosGetInfoBlocks(NULL, &pib); + return (int)pib->pib_ulpid; +} +#endif /* -** Open a file. +** Object used to represent a the shared memory area for a single log file. +** When multiple threads all reference the same log-summary, each thread has +** its own os2File object, but they all point to a single instance of this +** object. In other words, each log-summary is opened only once per process. +** +** os2ShmMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** pNext +** +** The following fields are read-only after the object is created: +** +** szRegion +** hLockFile +** shmBaseName +** +** Either os2ShmNode.mutex must be held or os2ShmNode.nRef==0 and +** os2ShmMutexHeld() is true when reading or writing any other field +** in this structure. +** */ -static int os2Open( - sqlite3_vfs *pVfs, /* Not used */ - const char *zName, /* Name of the file */ - sqlite3_file *id, /* Write the SQLite file handle here */ - int flags, /* Open mode flags */ - int *pOutFlags /* Status return flags */ -){ - HFILE h; - ULONG ulFileAttribute = FILE_NORMAL; - ULONG ulOpenFlags = 0; - ULONG ulOpenMode = 0; - os2File *pFile = (os2File*)id; - APIRET rc = NO_ERROR; - ULONG ulAction; - char *zNameCp; - char zTmpname[CCHMAXPATH+1]; /* Buffer to hold name of temp file */ - - /* If the second argument to this function is NULL, generate a - ** temporary file name to use - */ - if( !zName ){ - int rc = getTempname(CCHMAXPATH+1, zTmpname); - if( rc!=SQLITE_OK ){ - return rc; - } - zName = zTmpname; - } - +struct os2ShmNode { + sqlite3_mutex *mutex; /* Mutex to access this object */ + os2ShmNode *pNext; /* Next in list of all os2ShmNode objects */ - memset( pFile, 0, sizeof(*pFile) ); + int szRegion; /* Size of shared-memory regions */ - OSTRACE(( "OPEN want %d\n", flags )); + int nRegion; /* Size of array apRegion */ + void **apRegion; /* Array of pointers to shared-memory regions */ - if( flags & SQLITE_OPEN_READWRITE ){ - ulOpenMode |= OPEN_ACCESS_READWRITE; - OSTRACE(( "OPEN read/write\n" )); - }else{ - ulOpenMode |= OPEN_ACCESS_READONLY; - OSTRACE(( "OPEN read only\n" )); - } + int nRef; /* Number of os2ShmLink objects pointing to this */ + os2ShmLink *pFirst; /* First os2ShmLink object pointing to this */ - if( flags & SQLITE_OPEN_CREATE ){ - ulOpenFlags |= OPEN_ACTION_OPEN_IF_EXISTS | OPEN_ACTION_CREATE_IF_NEW; - OSTRACE(( "OPEN open new/create\n" )); - }else{ - ulOpenFlags |= OPEN_ACTION_OPEN_IF_EXISTS | OPEN_ACTION_FAIL_IF_NEW; - OSTRACE(( "OPEN open existing\n" )); - } + HFILE hLockFile; /* File used for inter-process memory locking */ + char shmBaseName[1]; /* Name of the memory object !!! must last !!! */ +}; - if( flags & SQLITE_OPEN_MAIN_DB ){ - ulOpenMode |= OPEN_SHARE_DENYNONE; - OSTRACE(( "OPEN share read/write\n" )); - }else{ - ulOpenMode |= OPEN_SHARE_DENYWRITE; - OSTRACE(( "OPEN share read only\n" )); - } - if( flags & SQLITE_OPEN_DELETEONCLOSE ){ - char pathUtf8[CCHMAXPATH]; -#ifdef NDEBUG /* when debugging we want to make sure it is deleted */ - ulFileAttribute = FILE_HIDDEN; +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** os2Shm.pShmNode +** os2Shm.id +** +** All other fields are read/write. The os2Shm.pShmNode->mutex must be held +** while accessing any read/write fields. +*/ +struct os2ShmLink { + os2ShmNode *pShmNode; /* The underlying os2ShmNode object */ + os2ShmLink *pNext; /* Next os2Shm with the same os2ShmNode */ + u32 sharedMask; /* Mask of shared locks held */ + u32 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_DEBUG + u8 id; /* Id of this connection with its os2ShmNode */ #endif - os2FullPathname( pVfs, zName, CCHMAXPATH, pathUtf8 ); - pFile->pathToDel = convertUtf8PathToCp( pathUtf8 ); - OSTRACE(( "OPEN hidden/delete on close file attributes\n" )); - }else{ - pFile->pathToDel = NULL; - OSTRACE(( "OPEN normal file attribute\n" )); - } - - /* always open in random access mode for possibly better speed */ - ulOpenMode |= OPEN_FLAGS_RANDOM; - ulOpenMode |= OPEN_FLAGS_FAIL_ON_ERROR; - ulOpenMode |= OPEN_FLAGS_NOINHERIT; - - zNameCp = convertUtf8PathToCp( zName ); - rc = DosOpen( (PSZ)zNameCp, - &h, - &ulAction, - 0L, - ulFileAttribute, - ulOpenFlags, - ulOpenMode, - (PEAOP2)NULL ); - free( zNameCp ); - if( rc != NO_ERROR ){ - OSTRACE(( "OPEN Invalid handle rc=%d: zName=%s, ulAction=%#lx, ulAttr=%#lx, ulFlags=%#lx, ulMode=%#lx\n", - rc, zName, ulAction, ulFileAttribute, ulOpenFlags, ulOpenMode )); - if( pFile->pathToDel ) - free( pFile->pathToDel ); - pFile->pathToDel = NULL; - if( flags & SQLITE_OPEN_READWRITE ){ - OSTRACE(( "OPEN %d Invalid handle\n", - ((flags | SQLITE_OPEN_READONLY) & ~SQLITE_OPEN_READWRITE) )); - return os2Open( pVfs, zName, id, - ((flags | SQLITE_OPEN_READONLY) & ~SQLITE_OPEN_READWRITE), - pOutFlags ); - }else{ - return SQLITE_CANTOPEN; - } - } +}; - if( pOutFlags ){ - *pOutFlags = flags & SQLITE_OPEN_READWRITE ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY; - } - pFile->pMethod = &os2IoMethod; - pFile->h = h; - OpenCounter(+1); - OSTRACE(( "OPEN %d pOutFlags=%d\n", pFile->h, pOutFlags )); - return SQLITE_OK; -} +/* +** A global list of all os2ShmNode objects. +** +** The os2ShmMutexHeld() must be true while reading or writing this list. +*/ +static os2ShmNode *os2ShmNodeList = NULL; /* -** Delete the named file. +** Constants used for locking */ -static int os2Delete( - sqlite3_vfs *pVfs, /* Not used on os2 */ - const char *zFilename, /* Name of file to delete */ - int syncDir /* Not used on os2 */ -){ - APIRET rc = NO_ERROR; - char *zFilenameCp = convertUtf8PathToCp( zFilename ); - SimulateIOError( return SQLITE_IOERR_DELETE ); - rc = DosDelete( (PSZ)zFilenameCp ); - free( zFilenameCp ); - OSTRACE(( "DELETE \"%s\"\n", zFilename )); - return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR_DELETE; -} +#ifdef SQLITE_OS2_NO_WAL_LOCK_FILE +#define OS2_SHM_BASE (PENDING_BYTE + 0x10000) /* first lock byte */ +#else +#define OS2_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#endif + +#define OS2_SHM_DMS (OS2_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ /* -** Check the existance and status of a file. +** Apply advisory locks for all n bytes beginning at ofst. */ -static int os2Access( - sqlite3_vfs *pVfs, /* Not used on os2 */ - const char *zFilename, /* Name of file to check */ - int flags, /* Type of test to make on this file */ - int *pOut /* Write results here */ +#define _SHM_UNLCK 1 /* no lock */ +#define _SHM_RDLCK 2 /* shared lock, no wait */ +#define _SHM_WRLCK 3 /* exlusive lock, no wait */ +#define _SHM_WRLCK_WAIT 4 /* exclusive lock, wait */ +static int os2ShmSystemLock( + os2ShmNode *pNode, /* Apply locks to this open shared-memory segment */ + int lockType, /* _SHM_UNLCK, _SHM_RDLCK, _SHM_WRLCK or _SHM_WRLCK_WAIT */ + int ofst, /* Offset to first byte to be locked/unlocked */ + int nByte /* Number of bytes to lock or unlock */ ){ - FILESTATUS3 fsts3ConfigInfo; - APIRET rc = NO_ERROR; - char *zFilenameCp = convertUtf8PathToCp( zFilename ); - - memset( &fsts3ConfigInfo, 0, sizeof(fsts3ConfigInfo) ); - rc = DosQueryPathInfo( (PSZ)zFilenameCp, FIL_STANDARD, - &fsts3ConfigInfo, sizeof(FILESTATUS3) ); - free( zFilenameCp ); - OSTRACE(( "ACCESS fsts3ConfigInfo.attrFile=%d flags=%d rc=%d\n", - fsts3ConfigInfo.attrFile, flags, rc )); - switch( flags ){ - case SQLITE_ACCESS_READ: - case SQLITE_ACCESS_EXISTS: - rc = (rc == NO_ERROR); - OSTRACE(( "ACCESS %s access of read and exists rc=%d\n", zFilename, rc)); - break; - case SQLITE_ACCESS_READWRITE: - rc = (rc == NO_ERROR) && ( (fsts3ConfigInfo.attrFile & FILE_READONLY) == 0 ); - OSTRACE(( "ACCESS %s access of read/write rc=%d\n", zFilename, rc )); + APIRET rc; + FILELOCK area; + ULONG mode, timeout; + + /* Access to the os2ShmNode object is serialized by the caller */ + assert( sqlite3_mutex_held(pNode->mutex) || pNode->nRef==0 ); + + mode = 1; /* shared lock */ + timeout = 0; /* no wait */ + area.lOffset = ofst; + area.lRange = nByte; + + switch( lockType ) { + case _SHM_WRLCK_WAIT: + timeout = (ULONG)-1; /* wait forever */ + case _SHM_WRLCK: + mode = 0; /* exclusive lock */ + case _SHM_RDLCK: + rc = DosSetFileLocks(pNode->hLockFile, + NULL, &area, timeout, mode); break; + /* case _SHM_UNLCK: */ default: - assert( !"Invalid flags argument" ); + rc = DosSetFileLocks(pNode->hLockFile, + &area, NULL, 0, 0); + break; } - *pOut = rc; - return SQLITE_OK; -} + + OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", + pNode->hLockFile, + rc==SQLITE_OK ? "ok" : "failed", + lockType==_SHM_UNLCK ? "Unlock" : "Lock", + rc)); + ERR_TRACE(rc, ("os2ShmSystemLock: %d %s\n", rc, pNode->shmBaseName)) -#ifndef SQLITE_OMIT_LOAD_EXTENSION -/* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ -/* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ -static void *os2DlOpen(sqlite3_vfs *pVfs, const char *zFilename){ - UCHAR loadErr[256]; - HMODULE hmod; - APIRET rc; - char *zFilenameCp = convertUtf8PathToCp(zFilename); - rc = DosLoadModule((PSZ)loadErr, sizeof(loadErr), zFilenameCp, &hmod); - free(zFilenameCp); - return rc != NO_ERROR ? 0 : (void*)hmod; + return ( rc == 0 ) ? SQLITE_OK : SQLITE_BUSY; } + /* -** A no-op since the error code is returned on the DosLoadModule call. -** os2Dlopen returns zero if DosLoadModule is not successful. +** Find an os2ShmNode in global list or allocate a new one, if not found. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. */ -static void os2DlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ -/* no-op */ -} -static void *os2DlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol){ - PFN pfn; - APIRET rc; - rc = DosQueryProcAddr((HMODULE)pHandle, 0L, zSymbol, &pfn); - if( rc != NO_ERROR ){ - /* if the symbol itself was not found, search again for the same - * symbol with an extra underscore, that might be needed depending - * on the calling convention */ - char _zSymbol[256] = "_"; - strncat(_zSymbol, zSymbol, 255); - rc = DosQueryProcAddr((HMODULE)pHandle, 0L, _zSymbol, &pfn); - } - return rc != NO_ERROR ? 0 : (void*)pfn; -} -static void os2DlClose(sqlite3_vfs *pVfs, void *pHandle){ - DosFreeModule((HMODULE)pHandle); -} -#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ - #define os2DlOpen 0 - #define os2DlError 0 - #define os2DlSym 0 - #define os2DlClose 0 +static int os2OpenSharedMemory( os2File *fd, int szRegion ) { + os2ShmLink *pLink; + os2ShmNode *pNode; + int cbShmName, rc = SQLITE_OK; + char shmName[CCHMAXPATH + 30]; +#ifndef SQLITE_OS2_NO_WAL_LOCK_FILE + ULONG action; #endif + + /* We need some additional space at the end to append the region number */ + cbShmName = sprintf(shmName, "\\SHAREMEM\\%s", fd->zFullPathCp ); + if( cbShmName >= CCHMAXPATH-8 ) + return SQLITE_IOERR_SHMOPEN; + /* Replace colon in file name to form a valid shared memory name */ + shmName[10+1] = '!'; -/* -** Write up to nBuf bytes of randomness into zBuf. -*/ -static int os2Randomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf ){ - int n = 0; -#if defined(SQLITE_TEST) - n = nBuf; - memset(zBuf, 0, nBuf); + /* Allocate link object (we free it later in case of failure) */ + pLink = sqlite3_malloc( sizeof(*pLink) ); + if( !pLink ) + return SQLITE_NOMEM; + + /* Access node list */ + os2ShmEnterMutex(); + + /* Find node by it's shared memory base name */ + for( pNode = os2ShmNodeList; + pNode && stricmp(shmName, pNode->shmBaseName) != 0; + pNode = pNode->pNext ) ; + + /* Not found: allocate a new node */ + if( !pNode ) { + pNode = sqlite3_malloc( sizeof(*pNode) + cbShmName ); + if( pNode ) { + memset(pNode, 0, sizeof(*pNode) ); + pNode->szRegion = szRegion; + pNode->hLockFile = (HFILE)-1; + strcpy(pNode->shmBaseName, shmName); + +#ifdef SQLITE_OS2_NO_WAL_LOCK_FILE + if( DosDupHandle(fd->h, &pNode->hLockFile) != 0 ) { #else - int sizeofULong = sizeof(ULONG); - if( (int)sizeof(DATETIME) <= nBuf - n ){ - DATETIME x; - DosGetDateTime(&x); - memcpy(&zBuf[n], &x, sizeof(x)); - n += sizeof(x); + sprintf(shmName, "%s-lck", fd->zFullPathCp); + if( DosOpen((PSZ)shmName, &pNode->hLockFile, &action, 0, FILE_NORMAL, + OPEN_ACTION_OPEN_IF_EXISTS | OPEN_ACTION_CREATE_IF_NEW, + OPEN_ACCESS_READWRITE | OPEN_SHARE_DENYNONE | + OPEN_FLAGS_NOINHERIT | OPEN_FLAGS_FAIL_ON_ERROR, + NULL) != 0 ) { +#endif + sqlite3_free(pNode); + rc = SQLITE_IOERR; + } else { + pNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( !pNode->mutex ) { + sqlite3_free(pNode); + rc = SQLITE_NOMEM; + } + } + } else { + rc = SQLITE_NOMEM; + } + + if( rc == SQLITE_OK ) { + pNode->pNext = os2ShmNodeList; + os2ShmNodeList = pNode; + } else { + pNode = NULL; + } + } else if( pNode->szRegion != szRegion ) { + rc = SQLITE_IOERR_SHMSIZE; + pNode = NULL; } - if( sizeofULong <= nBuf - n ){ - PPIB ppib; - DosGetInfoBlocks(NULL, &ppib); - memcpy(&zBuf[n], &ppib->pib_ulpid, sizeofULong); - n += sizeofULong; - } + if( pNode ) { + sqlite3_mutex_enter(pNode->mutex); - if( sizeofULong <= nBuf - n ){ - PTIB ptib; - DosGetInfoBlocks(&ptib, NULL); - memcpy(&zBuf[n], &ptib->tib_ptib2->tib2_ultid, sizeofULong); - n += sizeofULong; - } + memset(pLink, 0, sizeof(*pLink)); - /* if we still haven't filled the buffer yet the following will */ - /* grab everything once instead of making several calls for a single item */ - if( sizeofULong <= nBuf - n ){ - ULONG ulSysInfo[QSV_MAX]; - DosQuerySysInfo(1L, QSV_MAX, ulSysInfo, sizeofULong * QSV_MAX); + pLink->pShmNode = pNode; + pLink->pNext = pNode->pFirst; + pNode->pFirst = pLink; + pNode->nRef++; - memcpy(&zBuf[n], &ulSysInfo[QSV_MS_COUNT - 1], sizeofULong); - n += sizeofULong; + fd->pShmLink = pLink; - if( sizeofULong <= nBuf - n ){ - memcpy(&zBuf[n], &ulSysInfo[QSV_TIMER_INTERVAL - 1], sizeofULong); - n += sizeofULong; - } - if( sizeofULong <= nBuf - n ){ - memcpy(&zBuf[n], &ulSysInfo[QSV_TIME_LOW - 1], sizeofULong); - n += sizeofULong; - } - if( sizeofULong <= nBuf - n ){ - memcpy(&zBuf[n], &ulSysInfo[QSV_TIME_HIGH - 1], sizeofULong); - n += sizeofULong; - } - if( sizeofULong <= nBuf - n ){ - memcpy(&zBuf[n], &ulSysInfo[QSV_TOTAVAILMEM - 1], sizeofULong); - n += sizeofULong; - } + sqlite3_mutex_leave(pNode->mutex); + + } else { + /* Error occured. Free our link object. */ + sqlite3_free(pLink); } -#endif - return n; -} + os2ShmLeaveMutex(); -/* -** Sleep for a little while. Return the amount of time slept. -** The argument is the number of microseconds we want to sleep. -** The return value is the number of microseconds of sleep actually -** requested from the underlying operating system, a number which -** might be greater than or equal to the argument, but not less -** than the argument. -*/ -static int os2Sleep( sqlite3_vfs *pVfs, int microsec ){ - DosSleep( (microsec/1000) ); - return microsec; + ERR_TRACE(rc, ("os2OpenSharedMemory: %d %s\n", rc, fd->zFullPathCp)) + + return rc; } /* -** The following variable, if set to a non-zero value, becomes the result -** returned from sqlite3OsCurrentTime(). This is used for testing. +** Purge the os2ShmNodeList list of all entries with nRef==0. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_current_time = 0; -#endif +static void os2PurgeShmNodes( int deleteFlag ) { + os2ShmNode *pNode; + os2ShmNode **ppNode; -/* -** Find the current time (in Universal Coordinated Time). Write the -** current time and date as a Julian Day number into *prNow and -** return 0. Return 1 if the time and date cannot be found. -*/ -int os2CurrentTime( sqlite3_vfs *pVfs, double *prNow ){ - double now; - SHORT minute; /* needs to be able to cope with negative timezone offset */ - USHORT second, hour, - day, month, year; - DATETIME dt; - DosGetDateTime( &dt ); - second = (USHORT)dt.seconds; - minute = (SHORT)dt.minutes + dt.timezone; - hour = (USHORT)dt.hours; - day = (USHORT)dt.day; - month = (USHORT)dt.month; - year = (USHORT)dt.year; + os2ShmEnterMutex(); + + ppNode = &os2ShmNodeList; - /* Calculations from http://www.astro.keele.ac.uk/~rno/Astronomy/hjd.html - http://www.astro.keele.ac.uk/~rno/Astronomy/hjd-0.1.c */ - /* Calculate the Julian days */ - now = day - 32076 + - 1461*(year + 4800 + (month - 14)/12)/4 + - 367*(month - 2 - (month - 14)/12*12)/12 - - 3*((year + 4900 + (month - 14)/12)/100)/4; + while( *ppNode ) { + pNode = *ppNode; - /* Add the fractional hours, mins and seconds */ - now += (hour + 12.0)/24.0; - now += minute/1440.0; - now += second/86400.0; - *prNow = now; -#ifdef SQLITE_TEST - if( sqlite3_current_time ){ - *prNow = sqlite3_current_time/86400.0 + 2440587.5; - } + if( pNode->nRef == 0 ) { + *ppNode = pNode->pNext; + + if( pNode->apRegion ) { + /* Prevent other processes from resizing the shared memory */ + os2ShmSystemLock(pNode, _SHM_WRLCK_WAIT, OS2_SHM_DMS, 1); + + while( pNode->nRegion-- ) { +#ifdef SQLITE_DEBUG + int rc = +#endif + DosFreeMem(pNode->apRegion[pNode->nRegion]); + + OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", + (int)GetCurrentProcessId(), pNode->nRegion, + rc == 0 ? "ok" : "failed")); + } + + /* Allow other processes to resize the shared memory */ + os2ShmSystemLock(pNode, _SHM_UNLCK, OS2_SHM_DMS, 1); + + sqlite3_free(pNode->apRegion); + } + + DosClose(pNode->hLockFile); + +#ifndef SQLITE_OS2_NO_WAL_LOCK_FILE + if( deleteFlag ) { + char fileName[CCHMAXPATH]; + /* Skip "\\SHAREMEM\\" */ + sprintf(fileName, "%s-lck", pNode->shmBaseName + 10); + /* restore colon */ + fileName[1] = ':'; + + DosForceDelete(fileName); + } #endif - return 0; -} -static int os2GetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - return 0; + sqlite3_mutex_free(pNode->mutex); + + sqlite3_free(pNode); + + } else { + ppNode = &pNode->pNext; + } + } + + os2ShmLeaveMutex(); } /* -** Initialize and deinitialize the operating system interface. +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file id. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the bExtend parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. */ -SQLITE_API int sqlite3_os_init(void){ - static sqlite3_vfs os2Vfs = { - 1, /* iVersion */ - sizeof(os2File), /* szOsFile */ - CCHMAXPATH, /* mxPathname */ - 0, /* pNext */ - "os2", /* zName */ - 0, /* pAppData */ - - os2Open, /* xOpen */ - os2Delete, /* xDelete */ - os2Access, /* xAccess */ - os2FullPathname, /* xFullPathname */ - os2DlOpen, /* xDlOpen */ - os2DlError, /* xDlError */ - os2DlSym, /* xDlSym */ - os2DlClose, /* xDlClose */ - os2Randomness, /* xRandomness */ - os2Sleep, /* xSleep */ - os2CurrentTime, /* xCurrentTime */ - os2GetLastError, /* xGetLastError */ - }; - sqlite3_vfs_register(&os2Vfs, 1); - initUconvObjects(); - return SQLITE_OK; -} -SQLITE_API int sqlite3_os_end(void){ - freeUconvObjects(); - return SQLITE_OK; -} +static int os2ShmMap( + sqlite3_file *id, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend block if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + PVOID pvTemp; + void **apRegion; + os2ShmNode *pNode; + int n, rc = SQLITE_OK; + char shmName[CCHMAXPATH]; + os2File *pFile = (os2File*)id; + + *pp = NULL; -#endif /* SQLITE_OS_OS2 */ + if( !pFile->pShmLink ) + rc = os2OpenSharedMemory( pFile, szRegion ); + + if( rc == SQLITE_OK ) { + pNode = pFile->pShmLink->pShmNode ; + + sqlite3_mutex_enter(pNode->mutex); + + assert( szRegion==pNode->szRegion ); -/************** End of os_os2.c **********************************************/ -/************** Begin file os_unix.c *****************************************/ -/* -** 2004 May 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains the VFS implementation for unix-like operating systems -** include Linux, MacOSX, *BSD, QNX, VxWorks, AIX, HPUX, and others. -** -** There are actually several different VFS implementations in this file. -** The differences are in the way that file locking is done. The default -** implementation uses Posix Advisory Locks. Alternative implementations -** use flock(), dot-files, various proprietary locking schemas, or simply -** skip locking all together. -** -** This source file is organized into divisions where the logic for various -** subfunctions is contained within the appropriate division. PLEASE -** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed -** in the correct division and should be clearly labeled. -** -** The layout of divisions is as follows: -** -** * General-purpose declarations and utility functions. -** * Unique file ID logic used by VxWorks. -** * Various locking primitive implementations (all except proxy locking): -** + for Posix Advisory Locks -** + for no-op locks -** + for dot-file locks -** + for flock() locking -** + for named semaphore locks (VxWorks only) -** + for AFP filesystem locks (MacOSX only) -** * sqlite3_file methods not associated with locking. -** * Definitions of sqlite3_io_methods objects for all locking -** methods plus "finder" functions for each locking method. -** * sqlite3_vfs method implementations. -** * Locking primitives for the proxy uber-locking-method. (MacOSX only) -** * Definitions of sqlite3_vfs objects for all locking methods -** plus implementations of sqlite3_os_init() and sqlite3_os_end(). -*/ -#if SQLITE_OS_UNIX /* This file is used on unix only */ + /* Unmapped region ? */ + if( iRegion >= pNode->nRegion ) { + /* Prevent other processes from resizing the shared memory */ + os2ShmSystemLock(pNode, _SHM_WRLCK_WAIT, OS2_SHM_DMS, 1); -/* -** There are various methods for file locking used for concurrency -** control: -** -** 1. POSIX locking (the default), -** 2. No locking, -** 3. Dot-file locking, -** 4. flock() locking, -** 5. AFP locking (OSX only), -** 6. Named POSIX semaphores (VXWorks only), -** 7. proxy locking. (OSX only) -** -** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE -** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic -** selection of the appropriate locking style based on the filesystem -** where the database is located. -*/ -#if !defined(SQLITE_ENABLE_LOCKING_STYLE) -# if defined(__APPLE__) -# define SQLITE_ENABLE_LOCKING_STYLE 1 -# else -# define SQLITE_ENABLE_LOCKING_STYLE 0 -# endif -#endif + apRegion = sqlite3_realloc( + pNode->apRegion, (iRegion + 1) * sizeof(apRegion[0])); -/* -** Define the OS_VXWORKS pre-processor macro to 1 if building on -** vxworks, or 0 otherwise. -*/ -#ifndef OS_VXWORKS -# if defined(__RTP__) || defined(_WRS_KERNEL) -# define OS_VXWORKS 1 -# else -# define OS_VXWORKS 0 -# endif -#endif + if( apRegion ) { + pNode->apRegion = apRegion; -/* -** These #defines should enable >2GB file support on Posix if the -** underlying operating system supports it. If the OS lacks -** large file support, these should be no-ops. -** -** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch -** on the compiler command line. This is necessary if you are compiling -** on a recent machine (ex: RedHat 7.2) but you want your code to work -** on an older machine (ex: RedHat 6.0). If you compile on RedHat 7.2 -** without this option, LFS is enable. But LFS does not exist in the kernel -** in RedHat 6.0, so the code won't work. Hence, for maximum binary -** portability you should omit LFS. -** -** The previous paragraph was written in 2005. (This paragraph is written -** on 2008-11-28.) These days, all Linux kernels support large files, so -** you should probably leave LFS enabled. But some embedded platforms might -** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. -*/ -#ifndef SQLITE_DISABLE_LFS -# define _LARGE_FILE 1 -# ifndef _FILE_OFFSET_BITS -# define _FILE_OFFSET_BITS 64 -# endif -# define _LARGEFILE_SOURCE 1 -#endif + while( pNode->nRegion <= iRegion ) { + sprintf(shmName, "%s-%u", + pNode->shmBaseName, pNode->nRegion); -/* -** standard include files. -*/ -#include -#include -#include -#include -#include -#include -#include + if( DosGetNamedSharedMem(&pvTemp, (PSZ)shmName, + PAG_READ | PAG_WRITE) != NO_ERROR ) { + if( !bExtend ) + break; -#if SQLITE_ENABLE_LOCKING_STYLE -# include -# if OS_VXWORKS -# include -# include -# else -# include -# include -# endif -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + if( DosAllocSharedMem(&pvTemp, (PSZ)shmName, szRegion, + PAG_READ | PAG_WRITE | PAG_COMMIT | OBJ_ANY) != NO_ERROR && + DosAllocSharedMem(&pvTemp, (PSZ)shmName, szRegion, + PAG_READ | PAG_WRITE | PAG_COMMIT) != NO_ERROR ) { + rc = SQLITE_NOMEM; + break; + } + } -#if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) -# include -#endif + apRegion[pNode->nRegion++] = pvTemp; + } -/* -** Allowed values of unixFile.fsFlags -*/ -#define SQLITE_FSFLAGS_IS_MSDOS 0x1 + /* zero out remaining entries */ + for( n = pNode->nRegion; n <= iRegion; n++ ) + pNode->apRegion[n] = NULL; -/* -** If we are to be thread-safe, include the pthreads header and define -** the SQLITE_UNIX_THREADS macro. -*/ -#if SQLITE_THREADSAFE -# define SQLITE_UNIX_THREADS 1 -#endif + /* Return this region (maybe zero) */ + *pp = pNode->apRegion[iRegion]; + } else { + rc = SQLITE_NOMEM; + } -/* -** Default permissions when creating a new file -*/ -#ifndef SQLITE_DEFAULT_FILE_PERMISSIONS -# define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 -#endif + /* Allow other processes to resize the shared memory */ + os2ShmSystemLock(pNode, _SHM_UNLCK, OS2_SHM_DMS, 1); + + } else { + /* Region has been mapped previously */ + *pp = pNode->apRegion[iRegion]; + } -/* - ** Default permissions when creating auto proxy dir - */ -#ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS -# define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 -#endif + sqlite3_mutex_leave(pNode->mutex); + } -/* -** Maximum supported path-length. -*/ -#define MAX_PATHNAME 512 + ERR_TRACE(rc, ("os2ShmMap: %s iRgn = %d, szRgn = %d, bExt = %d : %d\n", + pFile->zFullPathCp, iRegion, szRegion, bExtend, rc)) + + return rc; +} /* -** Only set the lastErrno if the error code is a real error and not -** a normal expected return code of SQLITE_BUSY or SQLITE_OK +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +** +** If there is no shared memory associated with the connection then this +** routine is a harmless no-op. */ -#define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) - -/* Forward references */ -typedef struct unixShm unixShm; /* Connection shared memory */ -typedef struct unixShmNode unixShmNode; /* Shared memory instance */ -typedef struct unixInodeInfo unixInodeInfo; /* An i-node */ -typedef struct UnixUnusedFd UnixUnusedFd; /* An unused file descriptor */ +static int os2ShmUnmap( + sqlite3_file *id, /* The underlying database file */ + int deleteFlag /* Delete shared-memory if true */ +){ + os2File *pFile = (os2File*)id; + os2ShmLink *pLink = pFile->pShmLink; + + if( pLink ) { + int nRef = -1; + os2ShmLink **ppLink; + os2ShmNode *pNode = pLink->pShmNode; -/* -** Sometimes, after a file handle is closed by SQLite, the file descriptor -** cannot be closed immediately. In these cases, instances of the following -** structure are used to store the file descriptor while waiting for an -** opportunity to either close or reuse it. -*/ -struct UnixUnusedFd { - int fd; /* File descriptor to close */ - int flags; /* Flags this file descriptor was opened with */ - UnixUnusedFd *pNext; /* Next unused file descriptor on same file */ -}; + sqlite3_mutex_enter(pNode->mutex); + + for( ppLink = &pNode->pFirst; + *ppLink && *ppLink != pLink; + ppLink = &(*ppLink)->pNext ) ; + + assert(*ppLink); + + if( *ppLink ) { + *ppLink = pLink->pNext; + nRef = --pNode->nRef; + } else { + ERR_TRACE(1, ("os2ShmUnmap: link not found ! %s\n", + pNode->shmBaseName)) + } + + pFile->pShmLink = NULL; + sqlite3_free(pLink); -/* -** The unixFile structure is subclass of sqlite3_file specific to the unix -** VFS implementations. -*/ -typedef struct unixFile unixFile; -struct unixFile { - sqlite3_io_methods const *pMethod; /* Always the first entry */ - unixInodeInfo *pInode; /* Info about locks on this inode */ - int h; /* The file descriptor */ - int dirfd; /* File descriptor for the directory */ - unsigned char eFileLock; /* The type of lock held on this fd */ - int lastErrno; /* The unix errno from last I/O error */ - void *lockingContext; /* Locking style specific state */ - UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ - int fileFlags; /* Miscellanous flags */ - const char *zPath; /* Name of the file */ - unixShm *pShm; /* Shared memory segment information */ - int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ -#if SQLITE_ENABLE_LOCKING_STYLE - int openFlags; /* The flags specified at open() */ -#endif -#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) - unsigned fsFlags; /* cached details from statfs() */ -#endif -#if OS_VXWORKS - int isDelete; /* Delete on close if true */ - struct vxworksFileId *pId; /* Unique file ID */ -#endif -#ifndef NDEBUG - /* The next group of variables are used to track whether or not the - ** transaction counter in bytes 24-27 of database files are updated - ** whenever any part of the database changes. An assertion fault will - ** occur if a file is updated without also updating the transaction - ** counter. This test is made to avoid new problems similar to the - ** one described by ticket #3584. - */ - unsigned char transCntrChng; /* True if the transaction counter changed */ - unsigned char dbUpdate; /* True if any part of database file changed */ - unsigned char inNormalWrite; /* True if in a normal write operation */ -#endif -#ifdef SQLITE_TEST - /* In test mode, increase the size of this structure a bit so that - ** it is larger than the struct CrashFile defined in test6.c. - */ - char aPadding[32]; -#endif -}; + sqlite3_mutex_leave(pNode->mutex); + + if( nRef == 0 ) + os2PurgeShmNodes( deleteFlag ); + } -/* -** The following macros define bits in unixFile.fileFlags -*/ -#define SQLITE_WHOLE_FILE_LOCKING 0x0001 /* Use whole-file locking */ + return SQLITE_OK; +} /* -** Include code that is common to all os_*.c files -*/ -/************** Include os_common.h in the middle of os_unix.c ***************/ -/************** Begin file os_common.h ***************************************/ -/* -** 2004 May 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains macros and a little bit of code that is common to -** all of the platform-specific files (os_*.c) and is #included into those -** files. +** Change the lock state for a shared-memory segment. ** -** This file should be #included by the os_*.c files only. It is not a -** general purpose header file. +** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** different here than in posix. In xShmLock(), one can go from unlocked +** to shared and back or from unlocked to exclusive and back. But one may +** not go from shared to exclusive or from exclusive to shared. */ -#ifndef _OS_COMMON_H_ -#define _OS_COMMON_H_ +static int os2ShmLock( + sqlite3_file *id, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + u32 mask; /* Mask of locks to take or release */ + int rc = SQLITE_OK; /* Result code */ + os2File *pFile = (os2File*)id; + os2ShmLink *p = pFile->pShmLink; /* The shared memory being locked */ + os2ShmLink *pX; /* For looping over all siblings */ + os2ShmNode *pShmNode = p->pShmNode; /* Our node */ + + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); -/* -** At least two bugs have slipped in because we changed the MEMORY_DEBUG -** macro to SQLITE_DEBUG and some older makefiles have not yet made the -** switch. The following code should catch this problem at compile-time. -*/ -#ifdef MEMORY_DEBUG -# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." -#endif + mask = (u32)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ + if( flags & SQLITE_SHM_UNLOCK ){ + u32 allMask = 0; /* Mask of locks held by siblings */ -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } - #if defined(__GNUC__) + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = os2ShmSystemLock(pShmNode, _SHM_UNLCK, ofst+OS2_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u32 allShared = 0; /* Union of locks held by connections other than "p" */ - #elif defined(_MSC_VER) + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = os2ShmSystemLock(pShmNode, _SHM_RDLCK, ofst+OS2_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } - #endif + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = os2ShmSystemLock(pShmNode, _SHM_WRLCK, ofst+OS2_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } + } -#elif (defined(__GNUC__) && defined(__x86_64__)) + sqlite3_mutex_leave(pShmNode->mutex); + + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", + p->id, (int)GetCurrentProcessId(), p->sharedMask, p->exclMask, + rc ? "failed" : "ok")); - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif (defined(__GNUC__) && defined(__ppc__)) + ERR_TRACE(rc, ("os2ShmLock: ofst = %d, n = %d, flags = 0x%x -> %d \n", + ofst, n, flags, rc)) + + return rc; +} - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void os2ShmBarrier( + sqlite3_file *id /* Database file holding the shared memory */ +){ + UNUSED_PARAMETER(id); + os2ShmEnterMutex(); + os2ShmLeaveMutex(); +} #else +# define os2ShmMap 0 +# define os2ShmLock 0 +# define os2ShmBarrier 0 +# define os2ShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ - #error Need implementation of sqlite3Hwtime() for your platform. - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for os2. +*/ +static const sqlite3_io_methods os2IoMethod = { + 2, /* iVersion */ + os2Close, /* xClose */ + os2Read, /* xRead */ + os2Write, /* xWrite */ + os2Truncate, /* xTruncate */ + os2Sync, /* xSync */ + os2FileSize, /* xFileSize */ + os2Lock, /* xLock */ + os2Unlock, /* xUnlock */ + os2CheckReservedLock, /* xCheckReservedLock */ + os2FileControl, /* xFileControl */ + os2SectorSize, /* xSectorSize */ + os2DeviceCharacteristics, /* xDeviceCharacteristics */ + os2ShmMap, /* xShmMap */ + os2ShmLock, /* xShmLock */ + os2ShmBarrier, /* xShmBarrier */ + os2ShmUnmap /* xShmUnmap */ +}; -#endif -#endif /* !defined(_HWTIME_H_) */ +/*************************************************************************** +** Here ends the I/O methods that form the sqlite3_io_methods object. +** +** The next block of code implements the VFS methods. +****************************************************************************/ -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ +/* +** Create a temporary file name in zBuf. zBuf must be big enough to +** hold at pVfs->mxPathname characters. +*/ +static int getTempname(int nBuf, char *zBuf ){ + static const char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + int i, j; + PSZ zTempPathCp; + char zTempPath[CCHMAXPATH]; + ULONG ulDriveNum, ulDriveMap; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. + */ + SimulateIOError( return SQLITE_IOERR ); + + if( sqlite3_temp_directory ) { + sqlite3_snprintf(CCHMAXPATH-30, zTempPath, "%s", sqlite3_temp_directory); + } else if( DosScanEnv( (PSZ)"TEMP", &zTempPathCp ) == NO_ERROR || + DosScanEnv( (PSZ)"TMP", &zTempPathCp ) == NO_ERROR || + DosScanEnv( (PSZ)"TMPDIR", &zTempPathCp ) == NO_ERROR ) { + char *zTempPathUTF = convertCpPathToUtf8( (char *)zTempPathCp ); + sqlite3_snprintf(CCHMAXPATH-30, zTempPath, "%s", zTempPathUTF); + free( zTempPathUTF ); + } else if( DosQueryCurrentDisk( &ulDriveNum, &ulDriveMap ) == NO_ERROR ) { + zTempPath[0] = (char)('A' + ulDriveNum - 1); + zTempPath[1] = ':'; + zTempPath[2] = '\0'; + } else { + zTempPath[0] = '\0'; + } + + /* Strip off a trailing slashes or backslashes, otherwise we would get * + * multiple (back)slashes which causes DosOpen() to fail. * + * Trailing spaces are not allowed, either. */ + j = sqlite3Strlen30(zTempPath); + while( j > 0 && ( zTempPath[j-1] == '\\' || zTempPath[j-1] == '/' || + zTempPath[j-1] == ' ' ) ){ + j--; + } + zTempPath[j] = '\0'; + + /* We use 20 bytes to randomize the name */ + sqlite3_snprintf(nBuf-22, zBuf, + "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath); + j = sqlite3Strlen30(zBuf); + sqlite3_randomness( 20, &zBuf[j] ); + for( i = 0; i < 20; i++, j++ ){ + zBuf[j] = zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + + OSTRACE(( "TEMP FILENAME: %s\n", zBuf )); + return SQLITE_OK; +} -static sqlite_uint64 g_start; -static sqlite_uint64 g_elapsed; -#define TIMER_START g_start=sqlite3Hwtime() -#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start -#define TIMER_ELAPSED g_elapsed -#else -#define TIMER_START -#define TIMER_END -#define TIMER_ELAPSED ((sqlite_uint64)0) -#endif /* -** If we compile with the SQLITE_TEST macro set, then the following block -** of code will give us the ability to simulate a disk I/O error. This -** is used for testing the I/O recovery logic. +** Turn a relative pathname into a full pathname. Write the full +** pathname into zFull[]. zFull[] will be at least pVfs->mxPathname +** bytes in size. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; -#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) -#define SimulateIOError(CODE) \ - if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ - || sqlite3_io_error_pending-- == 1 ) \ - { local_ioerr(); CODE; } -static void local_ioerr(){ - IOTRACE(("IOERR\n")); - sqlite3_io_error_hit++; - if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +static int os2FullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + char *zRelativeCp = convertUtf8PathToCp( zRelative ); + char zFullCp[CCHMAXPATH] = "\0"; + char *zFullUTF; + APIRET rc = DosQueryPathInfo( (PSZ)zRelativeCp, FIL_QUERYFULLNAME, + zFullCp, CCHMAXPATH ); + free( zRelativeCp ); + zFullUTF = convertCpPathToUtf8( zFullCp ); + sqlite3_snprintf( nFull, zFull, zFullUTF ); + free( zFullUTF ); + return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; } -#define SimulateDiskfullError(CODE) \ - if( sqlite3_diskfull_pending ){ \ - if( sqlite3_diskfull_pending == 1 ){ \ - local_ioerr(); \ - sqlite3_diskfull = 1; \ - sqlite3_io_error_hit = 1; \ - CODE; \ - }else{ \ - sqlite3_diskfull_pending--; \ - } \ - } -#else -#define SimulateIOErrorBenign(X) -#define SimulateIOError(A) -#define SimulateDiskfullError(A) -#endif + /* -** When testing, keep a count of the number of open files. +** Open a file. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; -#define OpenCounter(X) sqlite3_open_file_count+=(X) -#else -#define OpenCounter(X) +static int os2Open( + sqlite3_vfs *pVfs, /* Not used */ + const char *zName, /* Name of the file (UTF-8) */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HFILE h; + ULONG ulOpenFlags = 0; + ULONG ulOpenMode = 0; + ULONG ulAction = 0; + ULONG rc; + os2File *pFile = (os2File*)id; + const char *zUtf8Name = zName; + char *zNameCp; + char zTmpname[CCHMAXPATH]; + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +#ifndef NDEBUG + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int eType = (flags & 0xFFFFFF00); + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); #endif -#endif /* !defined(_OS_COMMON_H_) */ + UNUSED_PARAMETER(pVfs); + assert( id!=0 ); -/************** End of os_common.h *******************************************/ -/************** Continuing where we left off in os_unix.c ********************/ + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); -/* -** Define various macros that are missing from some systems. -*/ -#ifndef O_LARGEFILE -# define O_LARGEFILE 0 -#endif -#ifdef SQLITE_DISABLE_LFS -# undef O_LARGEFILE -# define O_LARGEFILE 0 -#endif -#ifndef O_NOFOLLOW -# define O_NOFOLLOW 0 -#endif -#ifndef O_BINARY -# define O_BINARY 0 -#endif + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + memset( pFile, 0, sizeof(*pFile) ); + pFile->h = (HFILE)-1; + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zUtf8Name ){ + assert(isDelete && !isOpenJournal); + rc = getTempname(CCHMAXPATH, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zUtf8Name = zTmpname; + } + + if( isReadWrite ){ + ulOpenMode |= OPEN_ACCESS_READWRITE; + }else{ + ulOpenMode |= OPEN_ACCESS_READONLY; + } + + /* Open in random access mode for possibly better speed. Allow full + ** sharing because file locks will provide exclusive access when needed. + ** The handle should not be inherited by child processes and we don't + ** want popups from the critical error handler. + */ + ulOpenMode |= OPEN_FLAGS_RANDOM | OPEN_SHARE_DENYNONE | + OPEN_FLAGS_NOINHERIT | OPEN_FLAGS_FAIL_ON_ERROR; + + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" + ** as it is usually understood. + */ + if( isExclusive ){ + /* Creates a new file, only if it does not already exist. */ + /* If the file exists, it fails. */ + ulOpenFlags |= OPEN_ACTION_CREATE_IF_NEW | OPEN_ACTION_FAIL_IF_EXISTS; + }else if( isCreate ){ + /* Open existing file, or create if it doesn't exist */ + ulOpenFlags |= OPEN_ACTION_CREATE_IF_NEW | OPEN_ACTION_OPEN_IF_EXISTS; + }else{ + /* Opens a file, only if it exists. */ + ulOpenFlags |= OPEN_ACTION_FAIL_IF_NEW | OPEN_ACTION_OPEN_IF_EXISTS; + } + + zNameCp = convertUtf8PathToCp( zUtf8Name ); + rc = DosOpen( (PSZ)zNameCp, + &h, + &ulAction, + 0L, + FILE_NORMAL, + ulOpenFlags, + ulOpenMode, + (PEAOP2)NULL ); + free( zNameCp ); + + if( rc != NO_ERROR ){ + OSTRACE(( "OPEN Invalid handle rc=%d: zName=%s, ulAction=%#lx, ulFlags=%#lx, ulMode=%#lx\n", + rc, zUtf8Name, ulAction, ulOpenFlags, ulOpenMode )); + + if( isReadWrite ){ + return os2Open( pVfs, zName, id, + ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags ); + }else{ + return SQLITE_CANTOPEN; + } + } + + if( pOutFlags ){ + *pOutFlags = isReadWrite ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY; + } + + os2FullPathname( pVfs, zUtf8Name, sizeof( zTmpname ), zTmpname ); + pFile->zFullPathCp = convertUtf8PathToCp( zTmpname ); + pFile->pMethod = &os2IoMethod; + pFile->flags = flags; + pFile->h = h; + + OpenCounter(+1); + OSTRACE(( "OPEN %d pOutFlags=%d\n", pFile->h, pOutFlags )); + return SQLITE_OK; +} /* -** The DJGPP compiler environment looks mostly like Unix, but it -** lacks the fcntl() system call. So redefine fcntl() to be something -** that always succeeds. This means that locking does not occur under -** DJGPP. But it is DOS - what did you expect? +** Delete the named file. */ -#ifdef __DJGPP__ -# define fcntl(A,B,C) 0 -#endif +static int os2Delete( + sqlite3_vfs *pVfs, /* Not used on os2 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on os2 */ +){ + APIRET rc; + char *zFilenameCp; + SimulateIOError( return SQLITE_IOERR_DELETE ); + zFilenameCp = convertUtf8PathToCp( zFilename ); + rc = DosDelete( (PSZ)zFilenameCp ); + free( zFilenameCp ); + OSTRACE(( "DELETE \"%s\"\n", zFilename )); + return (rc == NO_ERROR || + rc == ERROR_FILE_NOT_FOUND || + rc == ERROR_PATH_NOT_FOUND ) ? SQLITE_OK : SQLITE_IOERR_DELETE; +} /* -** The threadid macro resolves to the thread-id or to 0. Used for -** testing and debugging only. +** Check the existance and status of a file. */ -#if SQLITE_THREADSAFE -#define threadid pthread_self() -#else -#define threadid 0 -#endif +static int os2Access( + sqlite3_vfs *pVfs, /* Not used on os2 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pOut /* Write results here */ +){ + APIRET rc; + FILESTATUS3 fsts3ConfigInfo; + char *zFilenameCp; + + UNUSED_PARAMETER(pVfs); + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + + zFilenameCp = convertUtf8PathToCp( zFilename ); + rc = DosQueryPathInfo( (PSZ)zFilenameCp, FIL_STANDARD, + &fsts3ConfigInfo, sizeof(FILESTATUS3) ); + free( zFilenameCp ); + OSTRACE(( "ACCESS fsts3ConfigInfo.attrFile=%d flags=%d rc=%d\n", + fsts3ConfigInfo.attrFile, flags, rc )); + + switch( flags ){ + case SQLITE_ACCESS_EXISTS: + /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file + ** as if it does not exist. + */ + if( fsts3ConfigInfo.cbFile == 0 ) + rc = ERROR_FILE_NOT_FOUND; + break; + case SQLITE_ACCESS_READ: + break; + case SQLITE_ACCESS_READWRITE: + if( fsts3ConfigInfo.attrFile & FILE_READONLY ) + rc = ERROR_ACCESS_DENIED; + break; + default: + rc = ERROR_FILE_NOT_FOUND; + assert( !"Invalid flags argument" ); + } + + *pOut = (rc == NO_ERROR); + OSTRACE(( "ACCESS %s flags %d: rc=%d\n", zFilename, flags, *pOut )); + + return SQLITE_OK; +} +#ifndef SQLITE_OMIT_LOAD_EXTENSION /* -** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the unixInodeInfo and -** vxworksFileId objects used by this file, all of which may be -** shared by multiple threads. -** -** Function unixMutexHeld() is used to assert() that the global mutex -** is held when required. This function is only used as part of assert() -** statements. e.g. -** -** unixEnterMutex() -** assert( unixMutexHeld() ); -** unixEnterLeave() +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. */ -static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +static void *os2DlOpen(sqlite3_vfs *pVfs, const char *zFilename){ + HMODULE hmod; + APIRET rc; + char *zFilenameCp = convertUtf8PathToCp(zFilename); + rc = DosLoadModule(NULL, 0, (PSZ)zFilenameCp, &hmod); + free(zFilenameCp); + return rc != NO_ERROR ? 0 : (void*)hmod; } -static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +/* +** A no-op since the error code is returned on the DosLoadModule call. +** os2Dlopen returns zero if DosLoadModule is not successful. +*/ +static void os2DlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ +/* no-op */ } -#ifdef SQLITE_DEBUG -static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +static void (*os2DlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ + PFN pfn; + APIRET rc; + rc = DosQueryProcAddr((HMODULE)pHandle, 0L, (PSZ)zSymbol, &pfn); + if( rc != NO_ERROR ){ + /* if the symbol itself was not found, search again for the same + * symbol with an extra underscore, that might be needed depending + * on the calling convention */ + char _zSymbol[256] = "_"; + strncat(_zSymbol, zSymbol, 254); + rc = DosQueryProcAddr((HMODULE)pHandle, 0L, (PSZ)_zSymbol, &pfn); + } + return rc != NO_ERROR ? 0 : (void(*)(void))pfn; } +static void os2DlClose(sqlite3_vfs *pVfs, void *pHandle){ + DosFreeModule((HMODULE)pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define os2DlOpen 0 + #define os2DlError 0 + #define os2DlSym 0 + #define os2DlClose 0 #endif -#ifdef SQLITE_DEBUG /* -** Helper function for printing out trace information from debugging -** binaries. This returns the string represetation of the supplied -** integer lock-type. +** Write up to nBuf bytes of randomness into zBuf. */ -static const char *azFileLock(int eFileLock){ - switch( eFileLock ){ - case NO_LOCK: return "NONE"; - case SHARED_LOCK: return "SHARED"; - case RESERVED_LOCK: return "RESERVED"; - case PENDING_LOCK: return "PENDING"; - case EXCLUSIVE_LOCK: return "EXCLUSIVE"; - } - return "ERROR"; +static int os2Randomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf ){ + int n = 0; +#if defined(SQLITE_TEST) + n = nBuf; + memset(zBuf, 0, nBuf); +#else + int i; + PPIB ppib; + PTIB ptib; + DATETIME dt; + static unsigned c = 0; + /* Ordered by variation probability */ + static ULONG svIdx[6] = { QSV_MS_COUNT, QSV_TIME_LOW, + QSV_MAXPRMEM, QSV_MAXSHMEM, + QSV_TOTAVAILMEM, QSV_TOTRESMEM }; + + /* 8 bytes; timezone and weekday don't increase the randomness much */ + if( (int)sizeof(dt)-3 <= nBuf - n ){ + c += 0x0100; + DosGetDateTime(&dt); + dt.year = (USHORT)((dt.year - 1900) | c); + memcpy(&zBuf[n], &dt, sizeof(dt)-3); + n += sizeof(dt)-3; + } + + /* 4 bytes; PIDs and TIDs are 16 bit internally, so combine them */ + if( (int)sizeof(ULONG) <= nBuf - n ){ + DosGetInfoBlocks(&ptib, &ppib); + *(PULONG)&zBuf[n] = MAKELONG(ppib->pib_ulpid, + ptib->tib_ptib2->tib2_ultid); + n += sizeof(ULONG); + } + + /* Up to 6 * 4 bytes; variables depend on the system state */ + for( i = 0; i < 6 && (int)sizeof(ULONG) <= nBuf - n; i++ ){ + DosQuerySysInfo(svIdx[i], svIdx[i], + (PULONG)&zBuf[n], sizeof(ULONG)); + n += sizeof(ULONG); + } +#endif + + return n; +} + +/* +** Sleep for a little while. Return the amount of time slept. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. +*/ +static int os2Sleep( sqlite3_vfs *pVfs, int microsec ){ + DosSleep( (microsec/1000) ); + return microsec; } + +/* +** The following variable, if set to a non-zero value, becomes the result +** returned from sqlite3OsCurrentTime(). This is used for testing. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; #endif -#ifdef SQLITE_LOCK_TRACE /* -** Print out information about all locking operations. +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. ** -** This routine is used for troubleshooting locks on multithreaded -** platforms. Enable by compiling with the -DSQLITE_LOCK_TRACE -** command-line option on the compiler. This code is normally -** turned off. +** On success, return 0. Return 1 if the time and date cannot be found. */ -static int lockTrace(int fd, int op, struct flock *p){ - char *zOpName, *zType; - int s; - int savedErrno; - if( op==F_GETLK ){ - zOpName = "GETLK"; - }else if( op==F_SETLK ){ - zOpName = "SETLK"; - }else{ - s = fcntl(fd, op, p); - sqlite3DebugPrintf("fcntl unknown %d %d %d\n", fd, op, s); - return s; - } - if( p->l_type==F_RDLCK ){ - zType = "RDLCK"; - }else if( p->l_type==F_WRLCK ){ - zType = "WRLCK"; - }else if( p->l_type==F_UNLCK ){ - zType = "UNLCK"; - }else{ - assert( 0 ); - } - assert( p->l_whence==SEEK_SET ); - s = fcntl(fd, op, p); - savedErrno = errno; - sqlite3DebugPrintf("fcntl %d %d %s %s %d %d %d %d\n", - threadid, fd, zOpName, zType, (int)p->l_start, (int)p->l_len, - (int)p->l_pid, s); - if( s==(-1) && op==F_SETLK && (p->l_type==F_RDLCK || p->l_type==F_WRLCK) ){ - struct flock l2; - l2 = *p; - fcntl(fd, F_GETLK, &l2); - if( l2.l_type==F_RDLCK ){ - zType = "RDLCK"; - }else if( l2.l_type==F_WRLCK ){ - zType = "WRLCK"; - }else if( l2.l_type==F_UNLCK ){ - zType = "UNLCK"; - }else{ - assert( 0 ); - } - sqlite3DebugPrintf("fcntl-failure-reason: %s %d %d %d\n", - zType, (int)l2.l_start, (int)l2.l_len, (int)l2.l_pid); - } - errno = savedErrno; - return s; -} -#define fcntl lockTrace -#endif /* SQLITE_LOCK_TRACE */ +static int os2CurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ +#ifdef SQLITE_TEST + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; +#endif + int year, month, datepart, timepart; + + DATETIME dt; + DosGetDateTime( &dt ); + + year = dt.year; + month = dt.month; + + /* Calculations from http://www.astro.keele.ac.uk/~rno/Astronomy/hjd.html + ** http://www.astro.keele.ac.uk/~rno/Astronomy/hjd-0.1.c + ** Calculate the Julian days + */ + datepart = (int)dt.day - 32076 + + 1461*(year + 4800 + (month - 14)/12)/4 + + 367*(month - 2 - (month - 14)/12*12)/12 - + 3*((year + 4900 + (month - 14)/12)/100)/4; + + /* Time in milliseconds, hours to noon added */ + timepart = 12*3600*1000 + dt.hundredths*10 + dt.seconds*1000 + + ((int)dt.minutes + dt.timezone)*60*1000 + dt.hours*3600*1000; + *piNow = (sqlite3_int64)datepart*86400*1000 + timepart; + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(pVfs); + return 0; +} /* -** This routine translates a standard POSIX errno code into something -** useful to the clients of the sqlite3 functions. Specifically, it is -** intended to translate a variety of "try again" errors into SQLITE_BUSY -** and a variety of "please close the file descriptor NOW" errors into -** SQLITE_IOERR -** -** Errors during initialization of locks, or file system support for locks, -** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. */ -static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { - switch (posixError) { - case 0: - return SQLITE_OK; - - case EAGAIN: - case ETIMEDOUT: - case EBUSY: - case EINTR: - case ENOLCK: - /* random NFS retry error, unless during file system support - * introspection, in which it actually means what it says */ - return SQLITE_BUSY; - - case EACCES: - /* EACCES is like EAGAIN during locking operations, but not any other time*/ - if( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || - (sqliteIOErr == SQLITE_IOERR_RDLOCK) || - (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ - return SQLITE_BUSY; - } - /* else fall through */ - case EPERM: - return SQLITE_PERM; - - case EDEADLK: - return SQLITE_IOERR_BLOCKED; - -#if EOPNOTSUPP!=ENOTSUP - case EOPNOTSUPP: - /* something went terribly awry, unless during file system support - * introspection, in which it actually means what it says */ -#endif -#ifdef ENOTSUP - case ENOTSUP: - /* invalid fd, unless during file system support introspection, in which - * it actually means what it says */ -#endif - case EIO: - case EBADF: - case EINVAL: - case ENOTCONN: - case ENODEV: - case ENXIO: - case ENOENT: - case ESTALE: - case ENOSYS: - /* these should force the client to close the file and reconnect */ - - default: - return sqliteIOErr; +static int os2CurrentTime( sqlite3_vfs *pVfs, double *prNow ){ + int rc; + sqlite3_int64 i; + rc = os2CurrentTimeInt64(pVfs, &i); + if( !rc ){ + *prNow = i/86400000.0; } + return rc; } - - -/****************************************************************************** -****************** Begin Unique File ID Utility Used By VxWorks *************** -** -** On most versions of unix, we can get a unique ID for a file by concatenating -** the device number and the inode number. But this does not work on VxWorks. -** On VxWorks, a unique file id must be based on the canonical filename. -** -** A pointer to an instance of the following structure can be used as a -** unique file ID in VxWorks. Each instance of this structure contains -** a copy of the canonical filename. There is also a reference count. -** The structure is reclaimed when the number of pointers to it drops to -** zero. -** -** There are never very many files open at one time and lookups are not -** a performance-critical path, so it is sufficient to put these -** structures on a linked list. -*/ -struct vxworksFileId { - struct vxworksFileId *pNext; /* Next in a list of them all */ - int nRef; /* Number of references to this one */ - int nName; /* Length of the zCanonicalName[] string */ - char *zCanonicalName; /* Canonical filename */ -}; - -#if OS_VXWORKS -/* -** All unique filenames are held on a linked list headed by this -** variable: -*/ -static struct vxworksFileId *vxworksFileList = 0; - /* -** Simplify a filename into its canonical form -** by making the following changes: +** The idea is that this function works like a combination of +** GetLastError() and FormatMessage() on windows (or errno and +** strerror_r() on unix). After an error is returned by an OS +** function, SQLite calls this function with zBuf pointing to +** a buffer of nBuf bytes. The OS layer should populate the +** buffer with a nul-terminated UTF-8 encoded error message +** describing the last IO error to have occurred within the calling +** thread. ** -** * removing any trailing and duplicate / -** * convert /./ into just / -** * convert /A/../ where A is any simple name into just / +** If the error message is too large for the supplied buffer, +** it should be truncated. The return value of xGetLastError +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). If non-zero is returned, +** then it is not necessary to include the nul-terminator character +** in the output buffer. ** -** Changes are made in-place. Return the new name length. +** Not supplying an error message will have no adverse effect +** on SQLite. It is fine to have an implementation that never +** returns an error message: ** -** The original filename is in z[0..n-1]. Return the number of -** characters in the simplified name. +** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +** assert(zBuf[0]=='\0'); +** return 0; +** } +** +** However if an error message is supplied, it will be incorporated +** by sqlite into the error message available to the user using +** sqlite3_errmsg(), possibly making IO errors easier to debug. */ -static int vxworksSimplifyName(char *z, int n){ - int i, j; - while( n>1 && z[n-1]=='/' ){ n--; } - for(i=j=0; i0 && z[j-1]!='/' ){ j--; } - if( j>0 ){ j--; } - i += 2; - continue; - } - } - z[j++] = z[i]; - } - z[j] = 0; - return j; +static int os2GetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + assert(zBuf[0]=='\0'); + return 0; } /* -** Find a unique file ID for the given absolute pathname. Return -** a pointer to the vxworksFileId object. This pointer is the unique -** file ID. -** -** The nRef field of the vxworksFileId object is incremented before -** the object is returned. A new vxworksFileId object is created -** and added to the global list if necessary. -** -** If a memory allocation error occurs, return NULL. +** Initialize and deinitialize the operating system interface. */ -static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ - struct vxworksFileId *pNew; /* search key and new file ID */ - struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ - int n; /* Length of zAbsoluteName string */ - - assert( zAbsoluteName[0]=='/' ); - n = (int)strlen(zAbsoluteName); - pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); - if( pNew==0 ) return 0; - pNew->zCanonicalName = (char*)&pNew[1]; - memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); - n = vxworksSimplifyName(pNew->zCanonicalName, n); - - /* Search for an existing entry that matching the canonical name. - ** If found, increment the reference count and return a pointer to - ** the existing file ID. - */ - unixEnterMutex(); - for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ - if( pCandidate->nName==n - && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 - ){ - sqlite3_free(pNew); - pCandidate->nRef++; - unixLeaveMutex(); - return pCandidate; - } - } +SQLITE_API int sqlite3_os_init(void){ + static sqlite3_vfs os2Vfs = { + 3, /* iVersion */ + sizeof(os2File), /* szOsFile */ + CCHMAXPATH, /* mxPathname */ + 0, /* pNext */ + "os2", /* zName */ + 0, /* pAppData */ - /* No match was found. We will make a new file ID */ - pNew->nRef = 1; - pNew->nName = n; - pNew->pNext = vxworksFileList; - vxworksFileList = pNew; - unixLeaveMutex(); - return pNew; + os2Open, /* xOpen */ + os2Delete, /* xDelete */ + os2Access, /* xAccess */ + os2FullPathname, /* xFullPathname */ + os2DlOpen, /* xDlOpen */ + os2DlError, /* xDlError */ + os2DlSym, /* xDlSym */ + os2DlClose, /* xDlClose */ + os2Randomness, /* xRandomness */ + os2Sleep, /* xSleep */ + os2CurrentTime, /* xCurrentTime */ + os2GetLastError, /* xGetLastError */ + os2CurrentTimeInt64, /* xCurrentTimeInt64 */ + 0, /* xSetSystemCall */ + 0, /* xGetSystemCall */ + 0 /* xNextSystemCall */ + }; + sqlite3_vfs_register(&os2Vfs, 1); + initUconvObjects(); +/* sqlite3OSTrace = 1; */ + return SQLITE_OK; } - -/* -** Decrement the reference count on a vxworksFileId object. Free -** the object when the reference count reaches zero. -*/ -static void vxworksReleaseFileId(struct vxworksFileId *pId){ - unixEnterMutex(); - assert( pId->nRef>0 ); - pId->nRef--; - if( pId->nRef==0 ){ - struct vxworksFileId **pp; - for(pp=&vxworksFileList; *pp && *pp!=pId; pp = &((*pp)->pNext)){} - assert( *pp==pId ); - *pp = pId->pNext; - sqlite3_free(pId); - } - unixLeaveMutex(); +SQLITE_API int sqlite3_os_end(void){ + freeUconvObjects(); + return SQLITE_OK; } -#endif /* OS_VXWORKS */ -/*************** End of Unique File ID Utility Used By VxWorks **************** -******************************************************************************/ +#endif /* SQLITE_OS_OS2 */ -/****************************************************************************** -*************************** Posix Advisory Locking **************************** -** -** POSIX advisory locks are broken by design. ANSI STD 1003.1 (1996) -** section 6.5.2.2 lines 483 through 490 specify that when a process -** sets or clears a lock, that operation overrides any prior locks set -** by the same process. It does not explicitly say so, but this implies -** that it overrides locks set by the same process using a different -** file descriptor. Consider this test case: -** -** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); -** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); -** -** Suppose ./file1 and ./file2 are really the same file (because -** one is a hard or symbolic link to the other) then if you set -** an exclusive lock on fd1, then try to get an exclusive lock -** on fd2, it works. I would have expected the second lock to -** fail since there was already a lock on the file due to fd1. -** But not so. Since both locks came from the same process, the -** second overrides the first, even though they were on different -** file descriptors opened on different file names. +/************** End of os_os2.c **********************************************/ +/************** Begin file os_unix.c *****************************************/ +/* +** 2004 May 22 ** -** This means that we cannot use POSIX locks to synchronize file access -** among competing threads of the same process. POSIX locks will work fine -** to synchronize access for threads in separate processes, but not -** threads within the same process. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** To work around the problem, SQLite has to manage file locks internally -** on its own. Whenever a new database is opened, we have to find the -** specific inode of the database file (the inode is determined by the -** st_dev and st_ino fields of the stat structure that fstat() fills in) -** and check for locks already existing on that inode. When locks are -** created or removed, we have to look at our own internal record of the -** locks to see if another thread has previously set a lock on that same -** inode. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** (Aside: The use of inode numbers as unique IDs does not work on VxWorks. -** For VxWorks, we have to use the alternative unique ID system based on -** canonical filename and implemented in the previous division.) +****************************************************************************** ** -** The sqlite3_file structure for POSIX is no longer just an integer file -** descriptor. It is now a structure that holds the integer file -** descriptor and a pointer to a structure that describes the internal -** locks on the corresponding inode. There is one locking structure -** per inode, so if the same inode is opened twice, both unixFile structures -** point to the same locking structure. The locking structure keeps -** a reference count (so we will know when to delete it) and a "cnt" -** field that tells us its internal lock status. cnt==0 means the -** file is unlocked. cnt==-1 means the file has an exclusive lock. -** cnt>0 means there are cnt shared locks on the file. +** This file contains the VFS implementation for unix-like operating systems +** include Linux, MacOSX, *BSD, QNX, VxWorks, AIX, HPUX, and others. ** -** Any attempt to lock or unlock a file first checks the locking -** structure. The fcntl() system call is only invoked to set a -** POSIX lock if the internal lock structure transitions between -** a locked and an unlocked state. +** There are actually several different VFS implementations in this file. +** The differences are in the way that file locking is done. The default +** implementation uses Posix Advisory Locks. Alternative implementations +** use flock(), dot-files, various proprietary locking schemas, or simply +** skip locking all together. ** -** But wait: there are yet more problems with POSIX advisory locks. +** This source file is organized into divisions where the logic for various +** subfunctions is contained within the appropriate division. PLEASE +** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed +** in the correct division and should be clearly labeled. ** -** If you close a file descriptor that points to a file that has locks, -** all locks on that file that are owned by the current process are -** released. To work around this problem, each unixInodeInfo object -** maintains a count of the number of pending locks on tha inode. -** When an attempt is made to close an unixFile, if there are -** other unixFile open on the same inode that are holding locks, the call -** to close() the file descriptor is deferred until all of the locks clear. -** The unixInodeInfo structure keeps a list of file descriptors that need to -** be closed and that list is walked (and cleared) when the last lock -** clears. +** The layout of divisions is as follows: ** -** Yet another problem: LinuxThreads do not play well with posix locks. +** * General-purpose declarations and utility functions. +** * Unique file ID logic used by VxWorks. +** * Various locking primitive implementations (all except proxy locking): +** + for Posix Advisory Locks +** + for no-op locks +** + for dot-file locks +** + for flock() locking +** + for named semaphore locks (VxWorks only) +** + for AFP filesystem locks (MacOSX only) +** * sqlite3_file methods not associated with locking. +** * Definitions of sqlite3_io_methods objects for all locking +** methods plus "finder" functions for each locking method. +** * sqlite3_vfs method implementations. +** * Locking primitives for the proxy uber-locking-method. (MacOSX only) +** * Definitions of sqlite3_vfs objects for all locking methods +** plus implementations of sqlite3_os_init() and sqlite3_os_end(). +*/ +#if SQLITE_OS_UNIX /* This file is used on unix only */ + +/* +** There are various methods for file locking used for concurrency +** control: ** -** Many older versions of linux use the LinuxThreads library which is -** not posix compliant. Under LinuxThreads, a lock created by thread -** A cannot be modified or overridden by a different thread B. -** Only thread A can modify the lock. Locking behavior is correct -** if the appliation uses the newer Native Posix Thread Library (NPTL) -** on linux - with NPTL a lock created by thread A can override locks -** in thread B. But there is no way to know at compile-time which -** threading library is being used. So there is no way to know at -** compile-time whether or not thread A can override locks on thread B. -** One has to do a run-time check to discover the behavior of the -** current process. +** 1. POSIX locking (the default), +** 2. No locking, +** 3. Dot-file locking, +** 4. flock() locking, +** 5. AFP locking (OSX only), +** 6. Named POSIX semaphores (VXWorks only), +** 7. proxy locking. (OSX only) ** -** SQLite used to support LinuxThreads. But support for LinuxThreads -** was dropped beginning with version 3.7.0. SQLite will still work with -** LinuxThreads provided that (1) there is no more than one connection -** per database file in the same process and (2) database connections -** do not move across threads. +** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE +** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic +** selection of the appropriate locking style based on the filesystem +** where the database is located. */ +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif /* -** An instance of the following structure serves as the key used -** to locate a particular unixInodeInfo object. +** Define the OS_VXWORKS pre-processor macro to 1 if building on +** vxworks, or 0 otherwise. */ -struct unixFileId { - dev_t dev; /* Device number */ -#if OS_VXWORKS - struct vxworksFileId *pId; /* Unique file ID for vxworks. */ -#else - ino_t ino; /* Inode number */ +#ifndef OS_VXWORKS +# if defined(__RTP__) || defined(_WRS_KERNEL) +# define OS_VXWORKS 1 +# else +# define OS_VXWORKS 0 +# endif #endif -}; /* -** An instance of the following structure is allocated for each open -** inode. Or, on LinuxThreads, there is one of these structures for -** each inode opened by each thread. +** These #defines should enable >2GB file support on Posix if the +** underlying operating system supports it. If the OS lacks +** large file support, these should be no-ops. ** -** A single inode can have multiple file descriptors, so each unixFile -** structure contains a pointer to an instance of this object and this -** object keeps a count of the number of unixFile pointing to it. +** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch +** on the compiler command line. This is necessary if you are compiling +** on a recent machine (ex: RedHat 7.2) but you want your code to work +** on an older machine (ex: RedHat 6.0). If you compile on RedHat 7.2 +** without this option, LFS is enable. But LFS does not exist in the kernel +** in RedHat 6.0, so the code won't work. Hence, for maximum binary +** portability you should omit LFS. +** +** The previous paragraph was written in 2005. (This paragraph is written +** on 2008-11-28.) These days, all Linux kernels support large files, so +** you should probably leave LFS enabled. But some embedded platforms might +** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. */ -struct unixInodeInfo { - struct unixFileId fileId; /* The lookup key */ - int nShared; /* Number of SHARED locks held */ - int eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ - int nRef; /* Number of pointers to this structure */ - unixShmNode *pShmNode; /* Shared memory associated with this inode */ - int nLock; /* Number of outstanding file locks */ - UnixUnusedFd *pUnused; /* Unused file descriptors to close */ - unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ - unixInodeInfo *pPrev; /* .... doubly linked */ -#if defined(SQLITE_ENABLE_LOCKING_STYLE) - unsigned long long sharedByte; /* for AFP simulated shared lock */ +#ifndef SQLITE_DISABLE_LFS +# define _LARGE_FILE 1 +# ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +# endif +# define _LARGEFILE_SOURCE 1 #endif -#if OS_VXWORKS - sem_t *pSem; /* Named POSIX semaphore */ - char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ + +/* +** standard include files. +*/ +#include +#include +#include +#include +#include +#include +#ifndef SQLITE_OMIT_WAL +#include +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE +# include +# if OS_VXWORKS +# include +# include +# else +# include +# include +# endif +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + +#if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +# include +#endif + +#ifdef HAVE_UTIME +# include #endif -}; /* -** A lists of all unixInodeInfo objects. +** Allowed values of unixFile.fsFlags */ -static unixInodeInfo *inodeList = 0; +#define SQLITE_FSFLAGS_IS_MSDOS 0x1 /* -** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. -** If all such file descriptors are closed without error, the list is -** cleared and SQLITE_OK returned. -** -** Otherwise, if an error occurs, then successfully closed file descriptor -** entries are removed from the list, and SQLITE_IOERR_CLOSE returned. -** not deleted and SQLITE_IOERR_CLOSE returned. -*/ -static int closePendingFds(unixFile *pFile){ - int rc = SQLITE_OK; - unixInodeInfo *pInode = pFile->pInode; - UnixUnusedFd *pError = 0; - UnixUnusedFd *p; - UnixUnusedFd *pNext; - for(p=pInode->pUnused; p; p=pNext){ - pNext = p->pNext; - if( close(p->fd) ){ - pFile->lastErrno = errno; - rc = SQLITE_IOERR_CLOSE; - p->pNext = pError; - pError = p; - }else{ - sqlite3_free(p); - } - } - pInode->pUnused = pError; - return rc; -} +** If we are to be thread-safe, include the pthreads header and define +** the SQLITE_UNIX_THREADS macro. +*/ +#if SQLITE_THREADSAFE +# define SQLITE_UNIX_THREADS 1 +#endif /* -** Release a unixInodeInfo structure previously allocated by findInodeInfo(). -** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** Default permissions when creating a new file */ -static void releaseInodeInfo(unixFile *pFile){ - unixInodeInfo *pInode = pFile->pInode; - assert( unixMutexHeld() ); - if( pInode ){ - pInode->nRef--; - if( pInode->nRef==0 ){ - assert( pInode->pShmNode==0 ); - closePendingFds(pFile); - if( pInode->pPrev ){ - assert( pInode->pPrev->pNext==pInode ); - pInode->pPrev->pNext = pInode->pNext; - }else{ - assert( inodeList==pInode ); - inodeList = pInode->pNext; - } - if( pInode->pNext ){ - assert( pInode->pNext->pPrev==pInode ); - pInode->pNext->pPrev = pInode->pPrev; - } - sqlite3_free(pInode); - } - } -} +#ifndef SQLITE_DEFAULT_FILE_PERMISSIONS +# define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 +#endif /* -** Given a file descriptor, locate the unixInodeInfo object that -** describes that file descriptor. Create a new one if necessary. The -** return value might be uninitialized if an error occurs. -** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. -** -** Return an appropriate error code. + ** Default permissions when creating auto proxy dir + */ +#ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +# define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 +#endif + +/* +** Maximum supported path-length. */ -static int findInodeInfo( - unixFile *pFile, /* Unix file with file desc used in the key */ - unixInodeInfo **ppInode /* Return the unixInodeInfo object here */ -){ - int rc; /* System call return code */ - int fd; /* The file descriptor for pFile */ - struct unixFileId fileId; /* Lookup key for the unixInodeInfo */ - struct stat statbuf; /* Low-level file information */ - unixInodeInfo *pInode = 0; /* Candidate unixInodeInfo object */ +#define MAX_PATHNAME 512 - assert( unixMutexHeld() ); +/* +** Only set the lastErrno if the error code is a real error and not +** a normal expected return code of SQLITE_BUSY or SQLITE_OK +*/ +#define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) - /* Get low-level information about the file that we can used to - ** create a unique name for the file. - */ - fd = pFile->h; - rc = fstat(fd, &statbuf); - if( rc!=0 ){ - pFile->lastErrno = errno; -#ifdef EOVERFLOW - if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; -#endif - return SQLITE_IOERR; - } +/* Forward references */ +typedef struct unixShm unixShm; /* Connection shared memory */ +typedef struct unixShmNode unixShmNode; /* Shared memory instance */ +typedef struct unixInodeInfo unixInodeInfo; /* An i-node */ +typedef struct UnixUnusedFd UnixUnusedFd; /* An unused file descriptor */ -#ifdef __APPLE__ - /* On OS X on an msdos filesystem, the inode number is reported - ** incorrectly for zero-size files. See ticket #3260. To work - ** around this problem (we consider it a bug in OS X, not SQLite) - ** we always increase the file size to 1 by writing a single byte - ** prior to accessing the inode number. The one byte written is - ** an ASCII 'S' character which also happens to be the first byte - ** in the header of every SQLite database. In this way, if there - ** is a race condition such that another thread has already populated - ** the first page of the database, no damage is done. - */ - if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ - rc = write(fd, "S", 1); - if( rc!=1 ){ - pFile->lastErrno = errno; - return SQLITE_IOERR; - } - rc = fstat(fd, &statbuf); - if( rc!=0 ){ - pFile->lastErrno = errno; - return SQLITE_IOERR; - } - } -#endif +/* +** Sometimes, after a file handle is closed by SQLite, the file descriptor +** cannot be closed immediately. In these cases, instances of the following +** structure are used to store the file descriptor while waiting for an +** opportunity to either close or reuse it. +*/ +struct UnixUnusedFd { + int fd; /* File descriptor to close */ + int flags; /* Flags this file descriptor was opened with */ + UnixUnusedFd *pNext; /* Next unused file descriptor on same file */ +}; - memset(&fileId, 0, sizeof(fileId)); - fileId.dev = statbuf.st_dev; +/* +** The unixFile structure is subclass of sqlite3_file specific to the unix +** VFS implementations. +*/ +typedef struct unixFile unixFile; +struct unixFile { + sqlite3_io_methods const *pMethod; /* Always the first entry */ + unixInodeInfo *pInode; /* Info about locks on this inode */ + int h; /* The file descriptor */ + int dirfd; /* File descriptor for the directory */ + unsigned char eFileLock; /* The type of lock held on this fd */ + unsigned char ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + int lastErrno; /* The unix errno from last I/O error */ + void *lockingContext; /* Locking style specific state */ + UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ + const char *zPath; /* Name of the file */ + unixShm *pShm; /* Shared memory segment information */ + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_ENABLE_LOCKING_STYLE + int openFlags; /* The flags specified at open() */ +#endif +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) + unsigned fsFlags; /* cached details from statfs() */ +#endif #if OS_VXWORKS - fileId.pId = pFile->pId; -#else - fileId.ino = statbuf.st_ino; + int isDelete; /* Delete on close if true */ + struct vxworksFileId *pId; /* Unique file ID */ #endif - pInode = inodeList; - while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ - pInode = pInode->pNext; - } - if( pInode==0 ){ - pInode = sqlite3_malloc( sizeof(*pInode) ); - if( pInode==0 ){ - return SQLITE_NOMEM; - } - memset(pInode, 0, sizeof(*pInode)); - memcpy(&pInode->fileId, &fileId, sizeof(fileId)); - pInode->nRef = 1; - pInode->pNext = inodeList; - pInode->pPrev = 0; - if( inodeList ) inodeList->pPrev = pInode; - inodeList = pInode; - }else{ - pInode->nRef++; - } - *ppInode = pInode; - return SQLITE_OK; -} - +#ifndef NDEBUG + /* The next group of variables are used to track whether or not the + ** transaction counter in bytes 24-27 of database files are updated + ** whenever any part of the database changes. An assertion fault will + ** occur if a file is updated without also updating the transaction + ** counter. This test is made to avoid new problems similar to the + ** one described by ticket #3584. + */ + unsigned char transCntrChng; /* True if the transaction counter changed */ + unsigned char dbUpdate; /* True if any part of database file changed */ + unsigned char inNormalWrite; /* True if in a normal write operation */ +#endif +#ifdef SQLITE_TEST + /* In test mode, increase the size of this structure a bit so that + ** it is larger than the struct CrashFile defined in test6.c. + */ + char aPadding[32]; +#endif +}; /* -** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** Allowed values for the unixFile.ctrlFlags bitmask: */ -static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc = SQLITE_OK; - int reserved = 0; - unixFile *pFile = (unixFile*)id; - - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ - assert( pFile ); - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ +/* +** Include code that is common to all os_*.c files +*/ +/************** Include os_common.h in the middle of os_unix.c ***************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ - /* Check if a thread in this process holds such a lock */ - if( pFile->pInode->eFileLock>SHARED_LOCK ){ - reserved = 1; - } +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif - /* Otherwise see if some other process holds it. - */ -#ifndef __DJGPP__ - if( !reserved ){ - struct flock lock; - lock.l_whence = SEEK_SET; - lock.l_start = RESERVED_BYTE; - lock.l_len = 1; - lock.l_type = F_WRLCK; - if (-1 == fcntl(pFile->h, F_GETLK, &lock)) { - int tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); - pFile->lastErrno = tErrno; - } else if( lock.l_type!=F_UNLCK ){ - reserved = 1; - } - } +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3OSTrace = 0; +#define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +#else +#define OSTRACE(X) #endif - - unixLeaveMutex(); - OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); - *pResOut = reserved; - return rc; -} +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of os_common.h ****************/ +/************** Begin file hwtime.h ******************************************/ /* -** Lock the file with the lock specified by parameter eFileLock - one -** of the following: +** 2008 May 27 ** -** (1) SHARED_LOCK -** (2) RESERVED_LOCK -** (3) PENDING_LOCK -** (4) EXCLUSIVE_LOCK +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Sometimes when requesting one lock state, additional lock states -** are inserted in between. The locking might fail on one of the later -** transitions leaving the lock state different from what it started but -** still short of its goal. The following chart shows the allowed -** transitions and the inserted intermediate states: +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** UNLOCKED -> SHARED -** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE -** RESERVED -> (PENDING) -> EXCLUSIVE -** PENDING -> EXCLUSIVE +****************************************************************************** ** -** This routine will only increase a lock. Use the sqlite3OsUnlock() -** routine to lower a locking level. +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. */ -static int unixLock(sqlite3_file *id, int eFileLock){ - /* The following describes the implementation of the various locks and - ** lock transitions in terms of the POSIX advisory shared and exclusive - ** lock primitives (called read-locks and write-locks below, to avoid - ** confusion with SQLite lock names). The algorithms are complicated - ** slightly in order to be compatible with windows systems simultaneously - ** accessing the same database file, in case that is ever required. - ** - ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved - ** byte', each single bytes at well known offsets, and the 'shared byte - ** range', a range of 510 bytes at a well known offset. - ** - ** To obtain a SHARED lock, a read-lock is obtained on the 'pending - ** byte'. If this is successful, a random byte from the 'shared byte - ** range' is read-locked and the lock on the 'pending byte' released. - ** - ** A process may only obtain a RESERVED lock after it has a SHARED lock. - ** A RESERVED lock is implemented by grabbing a write-lock on the - ** 'reserved byte'. - ** - ** A process may only obtain a PENDING lock after it has obtained a - ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock - ** on the 'pending byte'. This ensures that no new SHARED locks can be - ** obtained, but existing SHARED locks are allowed to persist. A process - ** does not have to obtain a RESERVED lock on the way to a PENDING lock. - ** This property is used by the algorithm for rolling back a journal file - ** after a crash. - ** - ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is - ** implemented by obtaining a write-lock on the entire 'shared byte - ** range'. Since all other locks require a read-lock on one of the bytes - ** within this range, this ensures that no other locks are held on the - ** database. - ** - ** The reason a single byte cannot be used instead of the 'shared byte - ** range' is that some versions of windows do not support read-locks. By - ** locking a random byte from a range, concurrent SHARED locks may exist - ** even if the locking primitive used is always a write-lock. - */ - int rc = SQLITE_OK; - unixFile *pFile = (unixFile*)id; - unixInodeInfo *pInode = pFile->pInode; - struct flock lock; - int s = 0; - int tErrno = 0; +#ifndef _HWTIME_H_ +#define _HWTIME_H_ - assert( pFile ); - OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, - azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pInode->eFileLock), pInode->nShared , getpid())); +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) - /* If there is already a lock of this type or more restrictive on the - ** unixFile, do nothing. Don't use the end_lock: exit path, as - ** unixEnterMutex() hasn't been called yet. - */ - if( pFile->eFileLock>=eFileLock ){ - OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, - azFileLock(eFileLock))); - return SQLITE_OK; - } + #if defined(__GNUC__) - /* Make sure the locking sequence is correct. - ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. - ** (3) A shared lock is always held when a reserve lock is requested. - */ - assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); - assert( eFileLock!=PENDING_LOCK ); - assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } - /* This mutex is needed because pFile->pInode is shared across threads - */ - unixEnterMutex(); - pInode = pFile->pInode; + #elif defined(_MSC_VER) - /* If some thread using this PID has a lock via a different unixFile* - ** handle that precludes the requested lock, return BUSY. - */ - if( (pFile->eFileLock!=pInode->eFileLock && - (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) - ){ - rc = SQLITE_BUSY; - goto end_lock; + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } } - /* If a SHARED lock is requested, and some thread using this PID already - ** has a SHARED or RESERVED lock, then increment reference counts and - ** return SQLITE_OK. - */ - if( eFileLock==SHARED_LOCK && - (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ - assert( eFileLock==SHARED_LOCK ); - assert( pFile->eFileLock==0 ); - assert( pInode->nShared>0 ); - pFile->eFileLock = SHARED_LOCK; - pInode->nShared++; - pInode->nLock++; - goto end_lock; - } + #endif +#elif (defined(__GNUC__) && defined(__x86_64__)) - /* A PENDING lock is needed before acquiring a SHARED lock and before - ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will - ** be released. - */ - lock.l_len = 1L; - lock.l_whence = SEEK_SET; - if( eFileLock==SHARED_LOCK - || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockh, F_SETLK, &lock); - if( s==(-1) ){ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - goto end_lock; - } + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; } + +#elif (defined(__GNUC__) && defined(__ppc__)) + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } - /* If control gets to this point, then actually go ahead and make - ** operating system calls for the specified lock. - */ - if( eFileLock==SHARED_LOCK ){ - assert( pInode->nShared==0 ); - assert( pInode->eFileLock==0 ); +#else - /* Now get the read-lock */ - lock.l_start = SHARED_FIRST; - lock.l_len = SHARED_SIZE; - if( (s = fcntl(pFile->h, F_SETLK, &lock))==(-1) ){ - tErrno = errno; - } - /* Drop the temporary PENDING lock */ - lock.l_start = PENDING_BYTE; - lock.l_len = 1L; - lock.l_type = F_UNLCK; - if( fcntl(pFile->h, F_SETLK, &lock)!=0 ){ - if( s != -1 ){ - /* This could happen with a network mount */ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - goto end_lock; - } - } - if( s==(-1) ){ - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - }else{ - pFile->eFileLock = SHARED_LOCK; - pInode->nLock++; - pInode->nShared = 1; - } - }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ - /* We are trying for an exclusive lock but another thread in this - ** same process is still holding a shared lock. */ - rc = SQLITE_BUSY; - }else{ - /* The request was for a RESERVED or EXCLUSIVE lock. It is - ** assumed that there is a SHARED or greater lock on the file - ** already. - */ - assert( 0!=pFile->eFileLock ); - lock.l_type = F_WRLCK; - switch( eFileLock ){ - case RESERVED_LOCK: - lock.l_start = RESERVED_BYTE; - break; - case EXCLUSIVE_LOCK: - lock.l_start = SHARED_FIRST; - lock.l_len = SHARED_SIZE; - break; - default: - assert(0); - } - s = fcntl(pFile->h, F_SETLK, &lock); - if( s==(-1) ){ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - } - } - + #error Need implementation of sqlite3Hwtime() for your platform. -#ifndef NDEBUG - /* Set up the transaction-counter change checking flags when - ** transitioning from a SHARED to a RESERVED lock. The change - ** from SHARED to RESERVED marks the beginning of a normal - ** write operation (not a hot journal rollback). + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. */ - if( rc==SQLITE_OK - && pFile->eFileLock<=SHARED_LOCK - && eFileLock==RESERVED_LOCK - ){ - pFile->transCntrChng = 0; - pFile->dbUpdate = 0; - pFile->inNormalWrite = 1; - } -#endif +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } +#endif - if( rc==SQLITE_OK ){ - pFile->eFileLock = eFileLock; - pInode->eFileLock = eFileLock; - }else if( eFileLock==EXCLUSIVE_LOCK ){ - pFile->eFileLock = PENDING_LOCK; - pInode->eFileLock = PENDING_LOCK; - } +#endif /* !defined(_HWTIME_H_) */ -end_lock: - unixLeaveMutex(); - OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), - rc==SQLITE_OK ? "ok" : "failed")); - return rc; -} +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in os_common.h ******************/ -/* -** Add the file descriptor used by file handle pFile to the corresponding -** pUnused list. -*/ -static void setPendingFd(unixFile *pFile){ - unixInodeInfo *pInode = pFile->pInode; - UnixUnusedFd *p = pFile->pUnused; - p->pNext = pInode->pUnused; - pInode->pUnused = p; - pFile->h = -1; - pFile->pUnused = 0; +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; } +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif /* -** Lower the locking level on file descriptor pFile to eFileLock. eFileLock -** must be either NO_LOCK or SHARED_LOCK. +** When testing, keep a count of the number of open files. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif + +#endif /* !defined(_OS_COMMON_H_) */ + +/************** End of os_common.h *******************************************/ +/************** Continuing where we left off in os_unix.c ********************/ + +/* +** Define various macros that are missing from some systems. +*/ +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifdef SQLITE_DISABLE_LFS +# undef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifndef O_NOFOLLOW +# define O_NOFOLLOW 0 +#endif +#ifndef O_BINARY +# define O_BINARY 0 +#endif + +/* +** The threadid macro resolves to the thread-id or to 0. Used for +** testing and debugging only. +*/ +#if SQLITE_THREADSAFE +#define threadid pthread_self() +#else +#define threadid 0 +#endif + +/* +** Different Unix systems declare open() in different ways. Same use +** open(const char*,int,mode_t). Others use open(const char*,int,...). +** The difference is important when using a pointer to the function. ** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. -** -** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED -** the byte range is divided into 2 parts and the first part is unlocked then -** set to a read lock, then the other part is simply unlocked. This works -** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to -** remove the write lock on a region when a read lock is set. +** The safest way to deal with the problem is to always use this wrapper +** which always has the same well-defined interface. */ -static int _posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ - unixFile *pFile = (unixFile*)id; - unixInodeInfo *pInode; - struct flock lock; - int rc = SQLITE_OK; - int h; - int tErrno; /* Error code from system call errors */ +static int posixOpen(const char *zFile, int flags, int mode){ + return open(zFile, flags, mode); +} - assert( pFile ); - OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, - pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, - getpid())); +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct unix_syscall { + const char *zName; /* Name of the sytem call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, +#define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) - assert( eFileLock<=SHARED_LOCK ); - if( pFile->eFileLock<=eFileLock ){ - return SQLITE_OK; - } - unixEnterMutex(); - h = pFile->h; - pInode = pFile->pInode; - assert( pInode->nShared!=0 ); - if( pFile->eFileLock>SHARED_LOCK ){ - assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); + { "close", (sqlite3_syscall_ptr)close, 0 }, +#define osClose ((int(*)(int))aSyscall[1].pCurrent) -#ifndef NDEBUG - /* When reducing a lock such that other processes can start - ** reading the database file again, make sure that the - ** transaction counter was updated if any part of the database - ** file changed. If the transaction counter is not updated, - ** other connections to the same file might not realize that - ** the file has changed and hence might not know to flush their - ** cache. The use of a stale cache can lead to database corruption. - */ -#if 0 - assert( pFile->inNormalWrite==0 - || pFile->dbUpdate==0 - || pFile->transCntrChng==1 ); + { "access", (sqlite3_syscall_ptr)access, 0 }, +#define osAccess ((int(*)(const char*,int))aSyscall[2].pCurrent) + + { "getcwd", (sqlite3_syscall_ptr)getcwd, 0 }, +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[3].pCurrent) + + { "stat", (sqlite3_syscall_ptr)stat, 0 }, +#define osStat ((int(*)(const char*,struct stat*))aSyscall[4].pCurrent) + +/* +** The DJGPP compiler environment looks mostly like Unix, but it +** lacks the fcntl() system call. So redefine fcntl() to be something +** that always succeeds. This means that locking does not occur under +** DJGPP. But it is DOS - what did you expect? +*/ +#ifdef __DJGPP__ + { "fstat", 0, 0 }, +#define osFstat(a,b,c) 0 +#else + { "fstat", (sqlite3_syscall_ptr)fstat, 0 }, +#define osFstat ((int(*)(int,struct stat*))aSyscall[5].pCurrent) #endif - pFile->inNormalWrite = 0; + + { "ftruncate", (sqlite3_syscall_ptr)ftruncate, 0 }, +#define osFtruncate ((int(*)(int,off_t))aSyscall[6].pCurrent) + + { "fcntl", (sqlite3_syscall_ptr)fcntl, 0 }, +#define osFcntl ((int(*)(int,int,...))aSyscall[7].pCurrent) + + { "read", (sqlite3_syscall_ptr)read, 0 }, +#define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) + +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE + { "pread", (sqlite3_syscall_ptr)pread, 0 }, +#else + { "pread", (sqlite3_syscall_ptr)0, 0 }, #endif +#define osPread ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[9].pCurrent) - /* downgrading to a shared lock on NFS involves clearing the write lock - ** before establishing the readlock - to avoid a race condition we downgrade - ** the lock in 2 blocks, so that part of the range will be covered by a - ** write lock until the rest is covered by a read lock: - ** 1: [WWWWW] - ** 2: [....W] - ** 3: [RRRRW] - ** 4: [RRRR.] +#if defined(USE_PREAD64) + { "pread64", (sqlite3_syscall_ptr)pread64, 0 }, +#else + { "pread64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) + + { "write", (sqlite3_syscall_ptr)write, 0 }, +#define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) + +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE + { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, +#else + { "pwrite", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite ((ssize_t(*)(int,const void*,size_t,off_t))\ + aSyscall[12].pCurrent) + +#if defined(USE_PREAD64) + { "pwrite64", (sqlite3_syscall_ptr)pwrite64, 0 }, +#else + { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ + aSyscall[13].pCurrent) + +#if SQLITE_ENABLE_LOCKING_STYLE + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#else + { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, +#else + { "fallocate", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFallocate ((int(*)(int,off_t,off_t))aSyscall[15].pCurrent) + +}; /* End of the overrideable system calls */ + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int unixSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL */ - if( eFileLock==SHARED_LOCK ){ - if( handleNFSUnlock ){ - off_t divSize = SHARED_SIZE - 1; - - lock.l_type = F_UNLCK; - lock.l_whence = SEEK_SET; - lock.l_start = SHARED_FIRST; - lock.l_len = divSize; - if( fcntl(h, F_SETLK, &lock)==(-1) ){ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - goto end_unlock; - } - lock.l_type = F_RDLCK; - lock.l_whence = SEEK_SET; - lock.l_start = SHARED_FIRST; - lock.l_len = divSize; - if( fcntl(h, F_SETLK, &lock)==(-1) ){ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - goto end_unlock; - } - lock.l_type = F_UNLCK; - lock.l_whence = SEEK_SET; - lock.l_start = SHARED_FIRST+divSize; - lock.l_len = SHARED_SIZE-divSize; - if( fcntl(h, F_SETLK, &lock)==(-1) ){ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - goto end_unlock; - } - }else{ - lock.l_type = F_RDLCK; - lock.l_whence = SEEK_SET; - lock.l_start = SHARED_FIRST; - lock.l_len = SHARED_SIZE; - if( fcntl(h, F_SETLK, &lock)==(-1) ){ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - goto end_unlock; - } - } - } - lock.l_type = F_UNLCK; - lock.l_whence = SEEK_SET; - lock.l_start = PENDING_BYTE; - lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); - if( fcntl(h, F_SETLK, &lock)!=(-1) ){ - pInode->eFileLock = SHARED_LOCK; - }else{ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + rc = SQLITE_OK; + for(i=0; inShared--; - if( pInode->nShared==0 ){ - lock.l_type = F_UNLCK; - lock.l_whence = SEEK_SET; - lock.l_start = lock.l_len = 0L; - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); - if( fcntl(h, F_SETLK, &lock)!=(-1) ){ - pInode->eFileLock = NO_LOCK; - }else{ - tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + for(i=0; ieFileLock = NO_LOCK; - pFile->eFileLock = NO_LOCK; - } - } - - /* Decrement the count of locks against this same file. When the - ** count reaches zero, close any other file descriptors whose close - ** was deferred because of outstanding locks. - */ - pInode->nLock--; - assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - int rc2 = closePendingFds(pFile); - if( rc==SQLITE_OK ){ - rc = rc2; + rc = SQLITE_OK; + if( pNewFunc==0 ) pNewFunc = aSyscall[i].pDefault; + aSyscall[i].pCurrent = pNewFunc; + break; } } } - -end_unlock: - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; return rc; } /* -** Lower the locking level on file descriptor pFile to eFileLock. eFileLock -** must be either NO_LOCK or SHARED_LOCK. -** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. +** Return the value of a system call. Return NULL if zName is not a +** recognized system call name. NULL is also returned if the system call +** is currently undefined. */ -static int unixUnlock(sqlite3_file *id, int eFileLock){ - return _posixUnlock(id, eFileLock, 0); +static sqlite3_syscall_ptr unixGetSystemCall( + sqlite3_vfs *pNotUsed, + const char *zName +){ + unsigned int i; + + UNUSED_PARAMETER(pNotUsed); + for(i=0; idirfd>=0 ){ - int err = close(pFile->dirfd); - if( err ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_DIR_CLOSE; - }else{ - pFile->dirfd=-1; - } - } - if( pFile->h>=0 ){ - int err = close(pFile->h); - if( err ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_CLOSE; - } - } -#if OS_VXWORKS - if( pFile->pId ){ - if( pFile->isDelete ){ - unlink(pFile->pId->zCanonicalName); - } - vxworksReleaseFileId(pFile->pId); - pFile->pId = 0; +static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ + int i = -1; + + UNUSED_PARAMETER(p); + if( zName ){ + for(i=0; ih)); - OpenCounter(-1); - sqlite3_free(pFile->pUnused); - memset(pFile, 0, sizeof(unixFile)); } - return SQLITE_OK; + for(i++; ipInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->pUnused list. It will be automatically closed - ** when the last lock is cleared. - */ - setPendingFd(pFile); - } - releaseInodeInfo(pFile); - rc = closeUnixFile(id); - unixLeaveMutex(); - } +static int robust_open(const char *z, int f, int m){ + int rc; + do{ rc = osOpen(z,f,m); }while( rc<0 && errno==EINTR ); return rc; } -/************** End of the posix advisory lock implementation ***************** -******************************************************************************/ - -/****************************************************************************** -****************************** No-op Locking ********************************** +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the unixInodeInfo and +** vxworksFileId objects used by this file, all of which may be +** shared by multiple threads. ** -** Of the various locking implementations available, this is by far the -** simplest: locking is ignored. No attempt is made to lock the database -** file for reading or writing. +** Function unixMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. ** -** This locking mode is appropriate for use on read-only databases -** (ex: databases that are burned into CD-ROM, for example.) It can -** also be used if the application employs some external mechanism to -** prevent simultaneous access of the same database by two or more -** database connections. But there is a serious risk of database -** corruption if this locking mode is used in situations where multiple -** database connections are accessing the same database file at the same -** time and one or more of those connections are writing. +** unixEnterMutex() +** assert( unixMutexHeld() ); +** unixEnterLeave() */ - -static int nolockCheckReservedLock(sqlite3_file *NotUsed, int *pResOut){ - UNUSED_PARAMETER(NotUsed); - *pResOut = 0; - return SQLITE_OK; +static void unixEnterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -static int nolockLock(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - return SQLITE_OK; +static void unixLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - return SQLITE_OK; +#ifdef SQLITE_DEBUG +static int unixMutexHeld(void) { + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } +#endif + +#ifdef SQLITE_DEBUG /* -** Close the file. +** Helper function for printing out trace information from debugging +** binaries. This returns the string represetation of the supplied +** integer lock-type. */ -static int nolockClose(sqlite3_file *id) { - return closeUnixFile(id); +static const char *azFileLock(int eFileLock){ + switch( eFileLock ){ + case NO_LOCK: return "NONE"; + case SHARED_LOCK: return "SHARED"; + case RESERVED_LOCK: return "RESERVED"; + case PENDING_LOCK: return "PENDING"; + case EXCLUSIVE_LOCK: return "EXCLUSIVE"; + } + return "ERROR"; } +#endif -/******************* End of the no-op lock implementation ********************* -******************************************************************************/ - -/****************************************************************************** -************************* Begin dot-file Locking ****************************** -** -** The dotfile locking implementation uses the existance of separate lock -** files in order to control access to the database. This works on just -** about every filesystem imaginable. But there are serious downsides: -** -** (1) There is zero concurrency. A single reader blocks all other -** connections from reading or writing the database. -** -** (2) An application crash or power loss can leave stale lock files -** sitting around that need to be cleared manually. -** -** Nevertheless, a dotlock is an appropriate locking mode for use if no -** other locking strategy is available. +#ifdef SQLITE_LOCK_TRACE +/* +** Print out information about all locking operations. ** -** Dotfile locking works by creating a file in the same directory as the -** database and with the same name but with a ".lock" extension added. -** The existance of a lock file implies an EXCLUSIVE lock. All other lock -** types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +** This routine is used for troubleshooting locks on multithreaded +** platforms. Enable by compiling with the -DSQLITE_LOCK_TRACE +** command-line option on the compiler. This code is normally +** turned off. */ +static int lockTrace(int fd, int op, struct flock *p){ + char *zOpName, *zType; + int s; + int savedErrno; + if( op==F_GETLK ){ + zOpName = "GETLK"; + }else if( op==F_SETLK ){ + zOpName = "SETLK"; + }else{ + s = osFcntl(fd, op, p); + sqlite3DebugPrintf("fcntl unknown %d %d %d\n", fd, op, s); + return s; + } + if( p->l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( p->l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( p->l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + assert( p->l_whence==SEEK_SET ); + s = osFcntl(fd, op, p); + savedErrno = errno; + sqlite3DebugPrintf("fcntl %d %d %s %s %d %d %d %d\n", + threadid, fd, zOpName, zType, (int)p->l_start, (int)p->l_len, + (int)p->l_pid, s); + if( s==(-1) && op==F_SETLK && (p->l_type==F_RDLCK || p->l_type==F_WRLCK) ){ + struct flock l2; + l2 = *p; + osFcntl(fd, F_GETLK, &l2); + if( l2.l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( l2.l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( l2.l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + sqlite3DebugPrintf("fcntl-failure-reason: %s %d %d %d\n", + zType, (int)l2.l_start, (int)l2.l_len, (int)l2.l_pid); + } + errno = savedErrno; + return s; +} +#undef osFcntl +#define osFcntl lockTrace +#endif /* SQLITE_LOCK_TRACE */ /* -** The file suffix added to the data base filename in order to create the -** lock file. +** Retry ftruncate() calls that fail due to EINTR */ -#define DOTLOCK_SUFFIX ".lock" +static int robust_ftruncate(int h, sqlite3_int64 sz){ + int rc; + do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); + return rc; +} /* -** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. -** -** In dotfile locking, either a lock exists or it does not. So in this -** variation of CheckReservedLock(), *pResOut is set to true if any lock -** is held on the file and false if the file is unlocked. +** This routine translates a standard POSIX errno code into something +** useful to the clients of the sqlite3 functions. Specifically, it is +** intended to translate a variety of "try again" errors into SQLITE_BUSY +** and a variety of "please close the file descriptor NOW" errors into +** SQLITE_IOERR +** +** Errors during initialization of locks, or file system support for locks, +** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ -static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { - int rc = SQLITE_OK; - int reserved = 0; - unixFile *pFile = (unixFile*)id; - - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - - assert( pFile ); +static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + switch (posixError) { +#if 0 + /* At one point this code was not commented out. In theory, this branch + ** should never be hit, as this function should only be called after + ** a locking-related function (i.e. fcntl()) has returned non-zero with + ** the value of errno as the first argument. Since a system call has failed, + ** errno should be non-zero. + ** + ** Despite this, if errno really is zero, we still don't want to return + ** SQLITE_OK. The system call failed, and *some* SQLite error should be + ** propagated back to the caller. Commenting this branch out means errno==0 + ** will be handled by the "default:" case below. + */ + case 0: + return SQLITE_OK; +#endif - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - /* Either this connection or some other connection in the same process - ** holds a lock on the file. No need to check further. */ - reserved = 1; - }else{ - /* The lock is held if and only if the lockfile exists */ - const char *zLockFile = (const char*)pFile->lockingContext; - reserved = access(zLockFile, 0)==0; + case EAGAIN: + case ETIMEDOUT: + case EBUSY: + case EINTR: + case ENOLCK: + /* random NFS retry error, unless during file system support + * introspection, in which it actually means what it says */ + return SQLITE_BUSY; + + case EACCES: + /* EACCES is like EAGAIN during locking operations, but not any other time*/ + if( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ + return SQLITE_BUSY; + } + /* else fall through */ + case EPERM: + return SQLITE_PERM; + + /* EDEADLK is only possible if a call to fcntl(F_SETLKW) is made. And + ** this module never makes such a call. And the code in SQLite itself + ** asserts that SQLITE_IOERR_BLOCKED is never returned. For these reasons + ** this case is also commented out. If the system does set errno to EDEADLK, + ** the default SQLITE_IOERR_XXX code will be returned. */ +#if 0 + case EDEADLK: + return SQLITE_IOERR_BLOCKED; +#endif + +#if EOPNOTSUPP!=ENOTSUP + case EOPNOTSUPP: + /* something went terribly awry, unless during file system support + * introspection, in which it actually means what it says */ +#endif +#ifdef ENOTSUP + case ENOTSUP: + /* invalid fd, unless during file system support introspection, in which + * it actually means what it says */ +#endif + case EIO: + case EBADF: + case EINVAL: + case ENOTCONN: + case ENODEV: + case ENXIO: + case ENOENT: + case ESTALE: + case ENOSYS: + /* these should force the client to close the file and reconnect */ + + default: + return sqliteIOErr; } - OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); - *pResOut = reserved; - return rc; } -/* -** Lock the file with the lock specified by parameter eFileLock - one -** of the following: -** -** (1) SHARED_LOCK -** (2) RESERVED_LOCK -** (3) PENDING_LOCK -** (4) EXCLUSIVE_LOCK -** -** Sometimes when requesting one lock state, additional lock states -** are inserted in between. The locking might fail on one of the later -** transitions leaving the lock state different from what it started but -** still short of its goal. The following chart shows the allowed -** transitions and the inserted intermediate states: + + +/****************************************************************************** +****************** Begin Unique File ID Utility Used By VxWorks *************** ** -** UNLOCKED -> SHARED -** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE -** RESERVED -> (PENDING) -> EXCLUSIVE -** PENDING -> EXCLUSIVE +** On most versions of unix, we can get a unique ID for a file by concatenating +** the device number and the inode number. But this does not work on VxWorks. +** On VxWorks, a unique file id must be based on the canonical filename. ** -** This routine will only increase a lock. Use the sqlite3OsUnlock() -** routine to lower a locking level. +** A pointer to an instance of the following structure can be used as a +** unique file ID in VxWorks. Each instance of this structure contains +** a copy of the canonical filename. There is also a reference count. +** The structure is reclaimed when the number of pointers to it drops to +** zero. ** -** With dotfile locking, we really only support state (4): EXCLUSIVE. -** But we track the other locking levels internally. +** There are never very many files open at one time and lookups are not +** a performance-critical path, so it is sufficient to put these +** structures on a linked list. */ -static int dotlockLock(sqlite3_file *id, int eFileLock) { - unixFile *pFile = (unixFile*)id; - int fd; - char *zLockFile = (char *)pFile->lockingContext; - int rc = SQLITE_OK; +struct vxworksFileId { + struct vxworksFileId *pNext; /* Next in a list of them all */ + int nRef; /* Number of references to this one */ + int nName; /* Length of the zCanonicalName[] string */ + char *zCanonicalName; /* Canonical filename */ +}; +#if OS_VXWORKS +/* +** All unique filenames are held on a linked list headed by this +** variable: +*/ +static struct vxworksFileId *vxworksFileList = 0; - /* If we have any lock, then the lock file already exists. All we have - ** to do is adjust our internal record of the lock level. - */ - if( pFile->eFileLock > NO_LOCK ){ - pFile->eFileLock = eFileLock; -#if !OS_VXWORKS - /* Always update the timestamp on the old file */ - utimes(zLockFile, NULL); -#endif - return SQLITE_OK; - } - - /* grab an exclusive lock */ - fd = open(zLockFile,O_RDONLY|O_CREAT|O_EXCL,0600); - if( fd<0 ){ - /* failed to open/create the file, someone else may have stolen the lock */ - int tErrno = errno; - if( EEXIST == tErrno ){ - rc = SQLITE_BUSY; - } else { - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; +/* +** Simplify a filename into its canonical form +** by making the following changes: +** +** * removing any trailing and duplicate / +** * convert /./ into just / +** * convert /A/../ where A is any simple name into just / +** +** Changes are made in-place. Return the new name length. +** +** The original filename is in z[0..n-1]. Return the number of +** characters in the simplified name. +*/ +static int vxworksSimplifyName(char *z, int n){ + int i, j; + while( n>1 && z[n-1]=='/' ){ n--; } + for(i=j=0; i0 && z[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; } } - return rc; - } - if( close(fd) ){ - pFile->lastErrno = errno; - rc = SQLITE_IOERR_CLOSE; + z[j++] = z[i]; } - - /* got it, set the type and return ok */ - pFile->eFileLock = eFileLock; - return rc; + z[j] = 0; + return j; } /* -** Lower the locking level on file descriptor pFile to eFileLock. eFileLock -** must be either NO_LOCK or SHARED_LOCK. +** Find a unique file ID for the given absolute pathname. Return +** a pointer to the vxworksFileId object. This pointer is the unique +** file ID. ** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. +** The nRef field of the vxworksFileId object is incremented before +** the object is returned. A new vxworksFileId object is created +** and added to the global list if necessary. ** -** When the locking level reaches NO_LOCK, delete the lock file. +** If a memory allocation error occurs, return NULL. */ -static int dotlockUnlock(sqlite3_file *id, int eFileLock) { - unixFile *pFile = (unixFile*)id; - char *zLockFile = (char *)pFile->lockingContext; +static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ + struct vxworksFileId *pNew; /* search key and new file ID */ + struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ + int n; /* Length of zAbsoluteName string */ - assert( pFile ); - OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); - assert( eFileLock<=SHARED_LOCK ); - - /* no-op if possible */ - if( pFile->eFileLock==eFileLock ){ - return SQLITE_OK; - } + assert( zAbsoluteName[0]=='/' ); + n = (int)strlen(zAbsoluteName); + pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); + if( pNew==0 ) return 0; + pNew->zCanonicalName = (char*)&pNew[1]; + memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); + n = vxworksSimplifyName(pNew->zCanonicalName, n); - /* To downgrade to shared, simply update our internal notion of the - ** lock state. No need to mess with the file on disk. + /* Search for an existing entry that matching the canonical name. + ** If found, increment the reference count and return a pointer to + ** the existing file ID. */ - if( eFileLock==SHARED_LOCK ){ - pFile->eFileLock = SHARED_LOCK; - return SQLITE_OK; - } - - /* To fully unlock the database, delete the lock file */ - assert( eFileLock==NO_LOCK ); - if( unlink(zLockFile) ){ - int rc = 0; - int tErrno = errno; - if( ENOENT != tErrno ){ - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - } - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + unixEnterMutex(); + for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ + if( pCandidate->nName==n + && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 + ){ + sqlite3_free(pNew); + pCandidate->nRef++; + unixLeaveMutex(); + return pCandidate; } - return rc; } - pFile->eFileLock = NO_LOCK; - return SQLITE_OK; + + /* No match was found. We will make a new file ID */ + pNew->nRef = 1; + pNew->nName = n; + pNew->pNext = vxworksFileList; + vxworksFileList = pNew; + unixLeaveMutex(); + return pNew; } /* -** Close a file. Make sure the lock has been released before closing. +** Decrement the reference count on a vxworksFileId object. Free +** the object when the reference count reaches zero. */ -static int dotlockClose(sqlite3_file *id) { - int rc; - if( id ){ - unixFile *pFile = (unixFile*)id; - dotlockUnlock(id, NO_LOCK); - sqlite3_free(pFile->lockingContext); +static void vxworksReleaseFileId(struct vxworksFileId *pId){ + unixEnterMutex(); + assert( pId->nRef>0 ); + pId->nRef--; + if( pId->nRef==0 ){ + struct vxworksFileId **pp; + for(pp=&vxworksFileList; *pp && *pp!=pId; pp = &((*pp)->pNext)){} + assert( *pp==pId ); + *pp = pId->pNext; + sqlite3_free(pId); } - rc = closeUnixFile(id); - return rc; + unixLeaveMutex(); } -/****************** End of the dot-file lock implementation ******************* +#endif /* OS_VXWORKS */ +/*************** End of Unique File ID Utility Used By VxWorks **************** ******************************************************************************/ + /****************************************************************************** -************************** Begin flock Locking ******************************** +*************************** Posix Advisory Locking **************************** ** -** Use the flock() system call to do file locking. +** POSIX advisory locks are broken by design. ANSI STD 1003.1 (1996) +** section 6.5.2.2 lines 483 through 490 specify that when a process +** sets or clears a lock, that operation overrides any prior locks set +** by the same process. It does not explicitly say so, but this implies +** that it overrides locks set by the same process using a different +** file descriptor. Consider this test case: ** -** flock() locking is like dot-file locking in that the various -** fine-grain locking levels supported by SQLite are collapsed into -** a single exclusive lock. In other words, SHARED, RESERVED, and -** PENDING locks are the same thing as an EXCLUSIVE lock. SQLite -** still works when you do this, but concurrency is reduced since -** only a single process can be reading the database at a time. +** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); +** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); ** -** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off or if -** compiling for VXWORKS. +** Suppose ./file1 and ./file2 are really the same file (because +** one is a hard or symbolic link to the other) then if you set +** an exclusive lock on fd1, then try to get an exclusive lock +** on fd2, it works. I would have expected the second lock to +** fail since there was already a lock on the file due to fd1. +** But not so. Since both locks came from the same process, the +** second overrides the first, even though they were on different +** file descriptors opened on different file names. +** +** This means that we cannot use POSIX locks to synchronize file access +** among competing threads of the same process. POSIX locks will work fine +** to synchronize access for threads in separate processes, but not +** threads within the same process. +** +** To work around the problem, SQLite has to manage file locks internally +** on its own. Whenever a new database is opened, we have to find the +** specific inode of the database file (the inode is determined by the +** st_dev and st_ino fields of the stat structure that fstat() fills in) +** and check for locks already existing on that inode. When locks are +** created or removed, we have to look at our own internal record of the +** locks to see if another thread has previously set a lock on that same +** inode. +** +** (Aside: The use of inode numbers as unique IDs does not work on VxWorks. +** For VxWorks, we have to use the alternative unique ID system based on +** canonical filename and implemented in the previous division.) +** +** The sqlite3_file structure for POSIX is no longer just an integer file +** descriptor. It is now a structure that holds the integer file +** descriptor and a pointer to a structure that describes the internal +** locks on the corresponding inode. There is one locking structure +** per inode, so if the same inode is opened twice, both unixFile structures +** point to the same locking structure. The locking structure keeps +** a reference count (so we will know when to delete it) and a "cnt" +** field that tells us its internal lock status. cnt==0 means the +** file is unlocked. cnt==-1 means the file has an exclusive lock. +** cnt>0 means there are cnt shared locks on the file. +** +** Any attempt to lock or unlock a file first checks the locking +** structure. The fcntl() system call is only invoked to set a +** POSIX lock if the internal lock structure transitions between +** a locked and an unlocked state. +** +** But wait: there are yet more problems with POSIX advisory locks. +** +** If you close a file descriptor that points to a file that has locks, +** all locks on that file that are owned by the current process are +** released. To work around this problem, each unixInodeInfo object +** maintains a count of the number of pending locks on tha inode. +** When an attempt is made to close an unixFile, if there are +** other unixFile open on the same inode that are holding locks, the call +** to close() the file descriptor is deferred until all of the locks clear. +** The unixInodeInfo structure keeps a list of file descriptors that need to +** be closed and that list is walked (and cleared) when the last lock +** clears. +** +** Yet another problem: LinuxThreads do not play well with posix locks. +** +** Many older versions of linux use the LinuxThreads library which is +** not posix compliant. Under LinuxThreads, a lock created by thread +** A cannot be modified or overridden by a different thread B. +** Only thread A can modify the lock. Locking behavior is correct +** if the appliation uses the newer Native Posix Thread Library (NPTL) +** on linux - with NPTL a lock created by thread A can override locks +** in thread B. But there is no way to know at compile-time which +** threading library is being used. So there is no way to know at +** compile-time whether or not thread A can override locks on thread B. +** One has to do a run-time check to discover the behavior of the +** current process. +** +** SQLite used to support LinuxThreads. But support for LinuxThreads +** was dropped beginning with version 3.7.0. SQLite will still work with +** LinuxThreads provided that (1) there is no more than one connection +** per database file in the same process and (2) database connections +** do not move across threads. */ -#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS /* -** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** An instance of the following structure serves as the key used +** to locate a particular unixInodeInfo object. */ -static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc = SQLITE_OK; - int reserved = 0; - unixFile *pFile = (unixFile*)id; - - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - - assert( pFile ); - - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - reserved = 1; - } - - /* Otherwise see if some other process holds it. */ - if( !reserved ){ - /* attempt to get the lock */ - int lrc = flock(pFile->h, LOCK_EX | LOCK_NB); - if( !lrc ){ - /* got the lock, unlock it */ - lrc = flock(pFile->h, LOCK_UN); - if ( lrc ) { - int tErrno = errno; - /* unlock failed with an error */ - lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(lrc) ){ - pFile->lastErrno = tErrno; - rc = lrc; - } - } - } else { - int tErrno = errno; - reserved = 1; - /* someone else might have it reserved */ - lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(lrc) ){ - pFile->lastErrno = tErrno; - rc = lrc; - } - } - } - OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); - -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ - rc = SQLITE_OK; - reserved=1; - } -#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ - *pResOut = reserved; - return rc; -} +struct unixFileId { + dev_t dev; /* Device number */ +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID for vxworks. */ +#else + ino_t ino; /* Inode number */ +#endif +}; /* -** Lock the file with the lock specified by parameter eFileLock - one -** of the following: -** -** (1) SHARED_LOCK -** (2) RESERVED_LOCK -** (3) PENDING_LOCK -** (4) EXCLUSIVE_LOCK +** An instance of the following structure is allocated for each open +** inode. Or, on LinuxThreads, there is one of these structures for +** each inode opened by each thread. ** -** Sometimes when requesting one lock state, additional lock states -** are inserted in between. The locking might fail on one of the later -** transitions leaving the lock state different from what it started but -** still short of its goal. The following chart shows the allowed -** transitions and the inserted intermediate states: +** A single inode can have multiple file descriptors, so each unixFile +** structure contains a pointer to an instance of this object and this +** object keeps a count of the number of unixFile pointing to it. +*/ +struct unixInodeInfo { + struct unixFileId fileId; /* The lookup key */ + int nShared; /* Number of SHARED locks held */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + int nRef; /* Number of pointers to this structure */ + unixShmNode *pShmNode; /* Shared memory associated with this inode */ + int nLock; /* Number of outstanding file locks */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ + unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ + unixInodeInfo *pPrev; /* .... doubly linked */ +#if SQLITE_ENABLE_LOCKING_STYLE + unsigned long long sharedByte; /* for AFP simulated shared lock */ +#endif +#if OS_VXWORKS + sem_t *pSem; /* Named POSIX semaphore */ + char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ +#endif +}; + +/* +** A lists of all unixInodeInfo objects. +*/ +static unixInodeInfo *inodeList = 0; + +/* ** -** UNLOCKED -> SHARED -** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE -** RESERVED -> (PENDING) -> EXCLUSIVE -** PENDING -> EXCLUSIVE +** This function - unixLogError_x(), is only ever called via the macro +** unixLogError(). ** -** flock() only really support EXCLUSIVE locks. We track intermediate -** lock states in the sqlite3_file structure, but all locks SHARED or -** above are really EXCLUSIVE locks and exclude all other processes from -** access the file. +** It is invoked after an error occurs in an OS function and errno has been +** set. It logs a message using sqlite3_log() containing the current value of +** errno and, if possible, the human-readable equivalent from strerror() or +** strerror_r(). ** -** This routine will only increase a lock. Use the sqlite3OsUnlock() -** routine to lower a locking level. +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed (e.g. "unlink", "open") and the the associated file-system path, +** if any. */ -static int flockLock(sqlite3_file *id, int eFileLock) { - int rc = SQLITE_OK; - unixFile *pFile = (unixFile*)id; +#define unixLogError(a,b,c) unixLogErrorAtLine(a,b,c,__LINE__) +static int unixLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char *zErr; /* Message from strerror() or equivalent */ + int iErrno = errno; /* Saved syscall error number */ - assert( pFile ); + /* If this is not a threadsafe build (SQLITE_THREADSAFE==0), then use + ** the strerror() function to obtain the human-readable error message + ** equivalent to errno. Otherwise, use strerror_r(). + */ +#if SQLITE_THREADSAFE && defined(HAVE_STRERROR_R) + char aErr[80]; + memset(aErr, 0, sizeof(aErr)); + zErr = aErr; + + /* If STRERROR_R_CHAR_P (set by autoconf scripts) or __USE_GNU is defined, + ** assume that the system provides the the GNU version of strerror_r() that + ** returns a pointer to a buffer containing the error message. That pointer + ** may point to aErr[], or it may point to some static storage somewhere. + ** Otherwise, assume that the system provides the POSIX version of + ** strerror_r(), which always writes an error message into aErr[]. + ** + ** If the code incorrectly assumes that it is the POSIX version that is + ** available, the error message will often be an empty string. Not a + ** huge problem. Incorrectly concluding that the GNU version is available + ** could lead to a segfault though. + */ +#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) + zErr = +# endif + strerror_r(iErrno, aErr, sizeof(aErr)-1); - /* if we already have a lock, it is exclusive. - ** Just adjust level and punt on outta here. */ - if (pFile->eFileLock > NO_LOCK) { - pFile->eFileLock = eFileLock; - return SQLITE_OK; - } - - /* grab an exclusive lock */ - - if (flock(pFile->h, LOCK_EX | LOCK_NB)) { - int tErrno = errno; - /* didn't get, must be busy */ - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - } else { - /* got it, set the type and return ok */ - pFile->eFileLock = eFileLock; - } - OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), - rc==SQLITE_OK ? "ok" : "failed")); -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ - rc = SQLITE_BUSY; - } -#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ - return rc; -} +#elif SQLITE_THREADSAFE + /* This is a threadsafe build, but strerror_r() is not available. */ + zErr = ""; +#else + /* Non-threadsafe build, use strerror(). */ + zErr = strerror(iErrno); +#endif + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + sqlite3_log(errcode, + "os_unix.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zErr + ); + + return errcode; +} /* -** Lower the locking level on file descriptor pFile to eFileLock. eFileLock -** must be either NO_LOCK or SHARED_LOCK. +** Close a file descriptor. ** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. +** We assume that close() almost always works, since it is only in a +** very sick application or on a very sick platform that it might fail. +** If it does fail, simply leak the file descriptor, but do log the +** error. +** +** Note that it is not safe to retry close() after EINTR since the +** file descriptor might have already been reused by another thread. +** So we don't even try to recover from an EINTR. Just log the error +** and move on. */ -static int flockUnlock(sqlite3_file *id, int eFileLock) { - unixFile *pFile = (unixFile*)id; - - assert( pFile ); - OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); - assert( eFileLock<=SHARED_LOCK ); - - /* no-op if possible */ - if( pFile->eFileLock==eFileLock ){ - return SQLITE_OK; - } - - /* shared can just be set because we always have an exclusive */ - if (eFileLock==SHARED_LOCK) { - pFile->eFileLock = eFileLock; - return SQLITE_OK; +static void robust_close(unixFile *pFile, int h, int lineno){ + if( osClose(h) ){ + unixLogErrorAtLine(SQLITE_IOERR_CLOSE, "close", + pFile ? pFile->zPath : 0, lineno); } - - /* no, really, unlock. */ - int rc = flock(pFile->h, LOCK_UN); - if (rc) { - int r, tErrno = errno; - r = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(r) ){ - pFile->lastErrno = tErrno; - } -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (r & SQLITE_IOERR) == SQLITE_IOERR ){ - r = SQLITE_BUSY; - } -#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ - - return r; - } else { - pFile->eFileLock = NO_LOCK; - return SQLITE_OK; +} + +/* +** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. +*/ +static void closePendingFds(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p; + UnixUnusedFd *pNext; + for(p=pInode->pUnused; p; p=pNext){ + pNext = p->pNext; + robust_close(pFile, p->fd, __LINE__); + sqlite3_free(p); } + pInode->pUnused = 0; } /* -** Close a file. +** Release a unixInodeInfo structure previously allocated by findInodeInfo(). +** +** The mutex entered using the unixEnterMutex() function must be held +** when this function is called. */ -static int flockClose(sqlite3_file *id) { - if( id ){ - flockUnlock(id, NO_LOCK); +static void releaseInodeInfo(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + assert( unixMutexHeld() ); + if( ALWAYS(pInode) ){ + pInode->nRef--; + if( pInode->nRef==0 ){ + assert( pInode->pShmNode==0 ); + closePendingFds(pFile); + if( pInode->pPrev ){ + assert( pInode->pPrev->pNext==pInode ); + pInode->pPrev->pNext = pInode->pNext; + }else{ + assert( inodeList==pInode ); + inodeList = pInode->pNext; + } + if( pInode->pNext ){ + assert( pInode->pNext->pPrev==pInode ); + pInode->pNext->pPrev = pInode->pPrev; + } + sqlite3_free(pInode); + } } - return closeUnixFile(id); } -#endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ - -/******************* End of the flock lock implementation ********************* -******************************************************************************/ - -/****************************************************************************** -************************ Begin Named Semaphore Locking ************************ +/* +** Given a file descriptor, locate the unixInodeInfo object that +** describes that file descriptor. Create a new one if necessary. The +** return value might be uninitialized if an error occurs. ** -** Named semaphore locking is only supported on VxWorks. +** The mutex entered using the unixEnterMutex() function must be held +** when this function is called. ** -** Semaphore locking is like dot-lock and flock in that it really only -** supports EXCLUSIVE locking. Only a single process can read or write -** the database file at a time. This reduces potential concurrency, but -** makes the lock implementation much easier. +** Return an appropriate error code. */ +static int findInodeInfo( + unixFile *pFile, /* Unix file with file desc used in the key */ + unixInodeInfo **ppInode /* Return the unixInodeInfo object here */ +){ + int rc; /* System call return code */ + int fd; /* The file descriptor for pFile */ + struct unixFileId fileId; /* Lookup key for the unixInodeInfo */ + struct stat statbuf; /* Low-level file information */ + unixInodeInfo *pInode = 0; /* Candidate unixInodeInfo object */ + + assert( unixMutexHeld() ); + + /* Get low-level information about the file that we can used to + ** create a unique name for the file. + */ + fd = pFile->h; + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + pFile->lastErrno = errno; +#ifdef EOVERFLOW + if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; +#endif + return SQLITE_IOERR; + } + +#ifdef __APPLE__ + /* On OS X on an msdos filesystem, the inode number is reported + ** incorrectly for zero-size files. See ticket #3260. To work + ** around this problem (we consider it a bug in OS X, not SQLite) + ** we always increase the file size to 1 by writing a single byte + ** prior to accessing the inode number. The one byte written is + ** an ASCII 'S' character which also happens to be the first byte + ** in the header of every SQLite database. In this way, if there + ** is a race condition such that another thread has already populated + ** the first page of the database, no damage is done. + */ + if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ + do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); + if( rc!=1 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR; + } + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR; + } + } +#endif + + memset(&fileId, 0, sizeof(fileId)); + fileId.dev = statbuf.st_dev; #if OS_VXWORKS + fileId.pId = pFile->pId; +#else + fileId.ino = statbuf.st_ino; +#endif + pInode = inodeList; + while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ + pInode = pInode->pNext; + } + if( pInode==0 ){ + pInode = sqlite3_malloc( sizeof(*pInode) ); + if( pInode==0 ){ + return SQLITE_NOMEM; + } + memset(pInode, 0, sizeof(*pInode)); + memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + pInode->nRef = 1; + pInode->pNext = inodeList; + pInode->pPrev = 0; + if( inodeList ) inodeList->pPrev = pInode; + inodeList = pInode; + }else{ + pInode->nRef++; + } + *ppInode = pInode; + return SQLITE_OK; +} + /* ** This routine checks if there is a RESERVED lock held on the specified @@ -24918,45 +25665,93 @@ static int flockClose(sqlite3_file *id) { ** to a non-zero value otherwise *pResOut is set to zero. The return value ** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { +static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ int rc = SQLITE_OK; int reserved = 0; unixFile *pFile = (unixFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - + assert( pFile ); + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ reserved = 1; } - - /* Otherwise see if some other process holds it. */ - if( !reserved ){ - sem_t *pSem = pFile->pInode->pSem; - struct stat statBuf; - if( sem_trywait(pSem)==-1 ){ - int tErrno = errno; - if( EAGAIN != tErrno ){ - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); - pFile->lastErrno = tErrno; - } else { - /* someone else has the lock when we are in NO_LOCK */ - reserved = (pFile->eFileLock < SHARED_LOCK); - } - }else{ - /* we could have it if we want it */ - sem_post(pSem); + /* Otherwise see if some other process holds it. + */ +#ifndef __DJGPP__ + if( !reserved && !pFile->pInode->bProcessLock ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = RESERVED_BYTE; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pFile->h, F_GETLK, &lock) ){ + rc = SQLITE_IOERR_CHECKRESERVEDLOCK; + pFile->lastErrno = errno; + } else if( lock.l_type!=F_UNLCK ){ + reserved = 1; } } - OSTRACE(("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved)); +#endif + + unixLeaveMutex(); + OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); *pResOut = reserved; return rc; } +/* +** Attempt to set a system-lock on the file pFile. The lock is +** described by pLock. +** +** If the pFile was opened read/write from unix-excl, then the only lock +** ever obtained is an exclusive lock, and it is obtained exactly once +** the first time any lock is attempted. All subsequent system locking +** operations become no-ops. Locking operations still happen internally, +** in order to coordinate access between separate database connections +** within this process, but all of that is handled in memory and the +** operating system does not participate. +** +** This function is a pass-through to fcntl(F_SETLK) if pFile is using +** any VFS other than "unix-excl" or if pFile is opened on "unix-excl" +** and is read-only. +** +** Zero is returned if the call completes successfully, or -1 if a call +** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). +*/ +static int unixFileLock(unixFile *pFile, struct flock *pLock){ + int rc; + unixInodeInfo *pInode = pFile->pInode; + assert( unixMutexHeld() ); + assert( pInode!=0 ); + if( ((pFile->ctrlFlags & UNIXFILE_EXCL)!=0 || pInode->bProcessLock) + && ((pFile->ctrlFlags & UNIXFILE_RDONLY)==0) + ){ + if( pInode->bProcessLock==0 ){ + struct flock lock; + assert( pInode->nLock==0 ); + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + lock.l_type = F_WRLCK; + rc = osFcntl(pFile->h, F_SETLK, &lock); + if( rc<0 ) return rc; + pInode->bProcessLock = 1; + pInode->nLock++; + }else{ + rc = 0; + } + }else{ + rc = osFcntl(pFile->h, F_SETLK, pLock); + } + return rc; +} + /* ** Lock the file with the lock specified by parameter eFileLock - one ** of the following: @@ -24978,231 +25773,603 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { ** RESERVED -> (PENDING) -> EXCLUSIVE ** PENDING -> EXCLUSIVE ** -** Semaphore locks only really support EXCLUSIVE locks. We track intermediate -** lock states in the sqlite3_file structure, but all locks SHARED or -** above are really EXCLUSIVE locks and exclude all other processes from -** access the file. -** ** This routine will only increase a lock. Use the sqlite3OsUnlock() ** routine to lower a locking level. */ -static int semLock(sqlite3_file *id, int eFileLock) { - unixFile *pFile = (unixFile*)id; - int fd; - sem_t *pSem = pFile->pInode->pSem; +static int unixLock(sqlite3_file *id, int eFileLock){ + /* The following describes the implementation of the various locks and + ** lock transitions in terms of the POSIX advisory shared and exclusive + ** lock primitives (called read-locks and write-locks below, to avoid + ** confusion with SQLite lock names). The algorithms are complicated + ** slightly in order to be compatible with windows systems simultaneously + ** accessing the same database file, in case that is ever required. + ** + ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** byte', each single bytes at well known offsets, and the 'shared byte + ** range', a range of 510 bytes at a well known offset. + ** + ** To obtain a SHARED lock, a read-lock is obtained on the 'pending + ** byte'. If this is successful, a random byte from the 'shared byte + ** range' is read-locked and the lock on the 'pending byte' released. + ** + ** A process may only obtain a RESERVED lock after it has a SHARED lock. + ** A RESERVED lock is implemented by grabbing a write-lock on the + ** 'reserved byte'. + ** + ** A process may only obtain a PENDING lock after it has obtained a + ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock + ** on the 'pending byte'. This ensures that no new SHARED locks can be + ** obtained, but existing SHARED locks are allowed to persist. A process + ** does not have to obtain a RESERVED lock on the way to a PENDING lock. + ** This property is used by the algorithm for rolling back a journal file + ** after a crash. + ** + ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is + ** implemented by obtaining a write-lock on the entire 'shared byte + ** range'. Since all other locks require a read-lock on one of the bytes + ** within this range, this ensures that no other locks are held on the + ** database. + ** + ** The reason a single byte cannot be used instead of the 'shared byte + ** range' is that some versions of windows do not support read-locks. By + ** locking a random byte from a range, concurrent SHARED locks may exist + ** even if the locking primitive used is always a write-lock. + */ int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode = pFile->pInode; + struct flock lock; + int tErrno = 0; - /* if we already have a lock, it is exclusive. - ** Just adjust level and punt on outta here. */ - if (pFile->eFileLock > NO_LOCK) { - pFile->eFileLock = eFileLock; - rc = SQLITE_OK; - goto sem_end_lock; + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pInode->eFileLock), pInode->nShared , getpid())); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; } - - /* lock semaphore now but bail out when already locked. */ - if( sem_trywait(pSem)==-1 ){ + + /* Make sure the locking sequence is correct. + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + unixEnterMutex(); + pInode = pFile->pInode; + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ rc = SQLITE_BUSY; - goto sem_end_lock; + goto end_lock; } - /* got it, set the type and return ok */ - pFile->eFileLock = eFileLock; + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto end_lock; + } - sem_end_lock: + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + lock.l_len = 1L; + lock.l_whence = SEEK_SET; + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLocklastErrno = tErrno; + } + goto end_lock; + } + } + + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( eFileLock==SHARED_LOCK ){ + assert( pInode->nShared==0 ); + assert( pInode->eFileLock==0 ); + assert( rc==SQLITE_OK ); + + /* Now get the read-lock */ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + } + + /* Drop the temporary PENDING lock */ + lock.l_start = PENDING_BYTE; + lock.l_len = 1L; + lock.l_type = F_UNLCK; + if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ + /* This could happen with a network mount */ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + } + + if( rc ){ + if( rc!=SQLITE_BUSY ){ + pFile->lastErrno = tErrno; + } + goto end_lock; + }else{ + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + assert( 0!=pFile->eFileLock ); + lock.l_type = F_WRLCK; + + assert( eFileLock==RESERVED_LOCK || eFileLock==EXCLUSIVE_LOCK ); + if( eFileLock==RESERVED_LOCK ){ + lock.l_start = RESERVED_BYTE; + lock.l_len = 1L; + }else{ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + } + + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( rc!=SQLITE_BUSY ){ + pFile->lastErrno = tErrno; + } + } + } + + +#ifndef NDEBUG + /* Set up the transaction-counter change checking flags when + ** transitioning from a SHARED to a RESERVED lock. The change + ** from SHARED to RESERVED marks the beginning of a normal + ** write operation (not a hot journal rollback). + */ + if( rc==SQLITE_OK + && pFile->eFileLock<=SHARED_LOCK + && eFileLock==RESERVED_LOCK + ){ + pFile->transCntrChng = 0; + pFile->dbUpdate = 0; + pFile->inNormalWrite = 1; + } +#endif + + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +end_lock: + unixLeaveMutex(); + OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); return rc; } +/* +** Add the file descriptor used by file handle pFile to the corresponding +** pUnused list. +*/ +static void setPendingFd(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p = pFile->pUnused; + p->pNext = pInode->pUnused; + pInode->pUnused = p; + pFile->h = -1; + pFile->pUnused = 0; +} + /* ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock ** must be either NO_LOCK or SHARED_LOCK. ** ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. +** +** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED +** the byte range is divided into 2 parts and the first part is unlocked then +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** remove the write lock on a region when a read lock is set. */ -static int semUnlock(sqlite3_file *id, int eFileLock) { +static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ unixFile *pFile = (unixFile*)id; - sem_t *pSem = pFile->pInode->pSem; + unixInodeInfo *pInode; + struct flock lock; + int rc = SQLITE_OK; + int h; assert( pFile ); - assert( pSem ); - OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + getpid())); + assert( eFileLock<=SHARED_LOCK ); - - /* no-op if possible */ - if( pFile->eFileLock==eFileLock ){ + if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - - /* shared can just be set because we always have an exclusive */ - if (eFileLock==SHARED_LOCK) { - pFile->eFileLock = eFileLock; - return SQLITE_OK; + unixEnterMutex(); + h = pFile->h; + pInode = pFile->pInode; + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifndef NDEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ +#if 0 + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); +#endif + pFile->inNormalWrite = 0; +#endif + + /* downgrading to a shared lock on NFS involves clearing the write lock + ** before establishing the readlock - to avoid a race condition we downgrade + ** the lock in 2 blocks, so that part of the range will be covered by a + ** write lock until the rest is covered by a read lock: + ** 1: [WWWWW] + ** 2: [....W] + ** 3: [RRRRW] + ** 4: [RRRR.] + */ + if( eFileLock==SHARED_LOCK ){ + +#if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE + (void)handleNFSUnlock; + assert( handleNFSUnlock==0 ); +#endif +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + if( handleNFSUnlock ){ + int tErrno; /* Error code from system call errors */ + off_t divSize = SHARED_SIZE - 1; + + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST+divSize; + lock.l_len = SHARED_SIZE-divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + }else +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + { + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + /* In theory, the call to unixFileLock() cannot fail because another + ** process is holding an incompatible lock. If it does, this + ** indicates that the other process is not following the locking + ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning + ** SQLITE_BUSY would confuse the upper layer (in practice it causes + ** an assert to fail). */ + rc = SQLITE_IOERR_RDLOCK; + pFile->lastErrno = errno; + goto end_unlock; + } + } + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = PENDING_BYTE; + lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = SHARED_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + pFile->lastErrno = errno; + goto end_unlock; + } } - - /* no, really unlock. */ - if ( sem_post(pSem)==-1 ) { - int rc, tErrno = errno; - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + if( eFileLock==NO_LOCK ){ + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + pInode->nShared--; + if( pInode->nShared==0 ){ + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = lock.l_len = 0L; + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = NO_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + pFile->lastErrno = errno; + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + + /* Decrement the count of locks against this same file. When the + ** count reaches zero, close any other file descriptors whose close + ** was deferred because of outstanding locks. + */ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ){ + closePendingFds(pFile); } - return rc; } - pFile->eFileLock = NO_LOCK; - return SQLITE_OK; + +end_unlock: + unixLeaveMutex(); + if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + return rc; } /* - ** Close a file. - */ -static int semClose(sqlite3_file *id) { - if( id ){ - unixFile *pFile = (unixFile*)id; - semUnlock(id, NO_LOCK); - assert( pFile ); - unixEnterMutex(); - releaseInodeInfo(pFile); - unixLeaveMutex(); - closeUnixFile(id); +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int unixUnlock(sqlite3_file *id, int eFileLock){ + return posixUnlock(id, eFileLock, 0); +} + +/* +** This function performs the parts of the "close file" operation +** common to all locking schemes. It closes the directory and file +** handles, if they are valid, and sets all fields of the unixFile +** structure to 0. +** +** It is *not* necessary to hold the mutex when this routine is called, +** even on VxWorks. A mutex will be acquired on VxWorks by the +** vxworksReleaseFileId() routine. +*/ +static int closeUnixFile(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; + if( pFile->dirfd>=0 ){ + robust_close(pFile, pFile->dirfd, __LINE__); + pFile->dirfd=-1; + } + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + pFile->h = -1; + } +#if OS_VXWORKS + if( pFile->pId ){ + if( pFile->isDelete ){ + unlink(pFile->pId->zCanonicalName); + } + vxworksReleaseFileId(pFile->pId); + pFile->pId = 0; } +#endif + OSTRACE(("CLOSE %-3d\n", pFile->h)); + OpenCounter(-1); + sqlite3_free(pFile->pUnused); + memset(pFile, 0, sizeof(unixFile)); return SQLITE_OK; } -#endif /* OS_VXWORKS */ /* -** Named semaphore locking is only available on VxWorks. -** -*************** End of the named semaphore lock implementation **************** -******************************************************************************/ +** Close a file. +*/ +static int unixClose(sqlite3_file *id){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile *)id; + unixUnlock(id, NO_LOCK); + unixEnterMutex(); + + /* unixFile.pInode is always valid here. Otherwise, a different close + ** routine (e.g. nolockClose()) would be called instead. + */ + assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); + if( ALWAYS(pFile->pInode) && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->pUnused list. It will be automatically closed + ** when the last lock is cleared. + */ + setPendingFd(pFile); + } + releaseInodeInfo(pFile); + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; +} +/************** End of the posix advisory lock implementation ***************** +******************************************************************************/ /****************************************************************************** -*************************** Begin AFP Locking ********************************* +****************************** No-op Locking ********************************** ** -** AFP is the Apple Filing Protocol. AFP is a network filesystem found -** on Apple Macintosh computers - both OS9 and OSX. +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. ** -** Third-party implementations of AFP are available. But this code here -** only works on OSX. +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. */ -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static int nolockCheckReservedLock(sqlite3_file *NotUsed, int *pResOut){ + UNUSED_PARAMETER(NotUsed); + *pResOut = 0; + return SQLITE_OK; +} +static int nolockLock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} +static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} + /* -** The afpLockingContext structure contains all afp lock specific state +** Close the file. */ -typedef struct afpLockingContext afpLockingContext; -struct afpLockingContext { - int reserved; - const char *dbPath; /* Name of the open file */ -}; +static int nolockClose(sqlite3_file *id) { + return closeUnixFile(id); +} -struct ByteRangeLockPB2 -{ - unsigned long long offset; /* offset to first byte to lock */ - unsigned long long length; /* nbr of bytes to lock */ - unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ - unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ - unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ - int fd; /* file desc to assoc this lock with */ -}; +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ -#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) +/****************************************************************************** +************************* Begin dot-file Locking ****************************** +** +** The dotfile locking implementation uses the existance of separate lock +** files in order to control access to the database. This works on just +** about every filesystem imaginable. But there are serious downsides: +** +** (1) There is zero concurrency. A single reader blocks all other +** connections from reading or writing the database. +** +** (2) An application crash or power loss can leave stale lock files +** sitting around that need to be cleared manually. +** +** Nevertheless, a dotlock is an appropriate locking mode for use if no +** other locking strategy is available. +** +** Dotfile locking works by creating a file in the same directory as the +** database and with the same name but with a ".lock" extension added. +** The existance of a lock file implies an EXCLUSIVE lock. All other lock +** types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +*/ /* -** This is a utility for setting or clearing a bit-range lock on an -** AFP filesystem. -** -** Return SQLITE_OK on success, SQLITE_BUSY on failure. +** The file suffix added to the data base filename in order to create the +** lock file. */ -static int afpSetLock( - const char *path, /* Name of the file to be locked or unlocked */ - unixFile *pFile, /* Open file descriptor on path */ - unsigned long long offset, /* First byte to be locked */ - unsigned long long length, /* Number of bytes to lock */ - int setLockFlag /* True to set lock. False to clear lock */ -){ - struct ByteRangeLockPB2 pb; - int err; - - pb.unLockFlag = setLockFlag ? 0 : 1; - pb.startEndFlag = 0; - pb.offset = offset; - pb.length = length; - pb.fd = pFile->h; - - OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", - (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), - offset, length)); - err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); - if ( err==-1 ) { - int rc; - int tErrno = errno; - OSTRACE(("AFPSETLOCK failed to fsctl() '%s' %d %s\n", - path, tErrno, strerror(tErrno))); -#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS - rc = SQLITE_BUSY; -#else - rc = sqliteErrorFromPosixError(tErrno, - setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); -#endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } - return rc; - } else { - return SQLITE_OK; - } -} +#define DOTLOCK_SUFFIX ".lock" /* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut ** to a non-zero value otherwise *pResOut is set to zero. The return value ** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** +** In dotfile locking, either a lock exists or it does not. So in this +** variation of CheckReservedLock(), *pResOut is set to true if any lock +** is held on the file and false if the file is unlocked. */ -static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ +static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { int rc = SQLITE_OK; int reserved = 0; unixFile *pFile = (unixFile*)id; - + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - if( context->reserved ){ - *pResOut = 1; - return SQLITE_OK; - } - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ - + /* Check if a thread in this process holds such a lock */ - if( pFile->pInode->eFileLock>SHARED_LOCK ){ + if( pFile->eFileLock>SHARED_LOCK ){ + /* Either this connection or some other connection in the same process + ** holds a lock on the file. No need to check further. */ reserved = 1; + }else{ + /* The lock is held if and only if the lockfile exists */ + const char *zLockFile = (const char*)pFile->lockingContext; + reserved = osAccess(zLockFile, 0)==0; } - - /* Otherwise see if some other process holds it. - */ - if( !reserved ){ - /* lock the RESERVED byte */ - int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); - if( SQLITE_OK==lrc ){ - /* if we succeeded in taking the reserved lock, unlock it to restore - ** the original state */ - lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); - } else { - /* if we failed to get the lock then someone else must have it */ - reserved = 1; - } - if( IS_LOCK_ERROR(lrc) ){ - rc=lrc; - } - } - - unixLeaveMutex(); - OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); - + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); *pResOut = reserved; return rc; } @@ -25230,8090 +26397,8293 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ ** ** This routine will only increase a lock. Use the sqlite3OsUnlock() ** routine to lower a locking level. +** +** With dotfile locking, we really only support state (4): EXCLUSIVE. +** But we track the other locking levels internally. */ -static int afpLock(sqlite3_file *id, int eFileLock){ - int rc = SQLITE_OK; +static int dotlockLock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - unixInodeInfo *pInode = pFile->pInode; - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - - assert( pFile ); - OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, - azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pInode->eFileLock), pInode->nShared , getpid())); + int fd; + char *zLockFile = (char *)pFile->lockingContext; + int rc = SQLITE_OK; - /* If there is already a lock of this type or more restrictive on the - ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as - ** unixEnterMutex() hasn't been called yet. + + /* If we have any lock, then the lock file already exists. All we have + ** to do is adjust our internal record of the lock level. */ - if( pFile->eFileLock>=eFileLock ){ - OSTRACE(("LOCK %d %s ok (already held) (afp)\n", pFile->h, - azFileLock(eFileLock))); + if( pFile->eFileLock > NO_LOCK ){ + pFile->eFileLock = eFileLock; + /* Always update the timestamp on the old file */ +#ifdef HAVE_UTIME + utime(zLockFile, NULL); +#else + utimes(zLockFile, NULL); +#endif return SQLITE_OK; } - - /* Make sure the locking sequence is correct - ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. - ** (3) A shared lock is always held when a reserve lock is requested. - */ - assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); - assert( eFileLock!=PENDING_LOCK ); - assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); - /* This mutex is needed because pFile->pInode is shared across threads - */ - unixEnterMutex(); - pInode = pFile->pInode; + /* grab an exclusive lock */ + fd = robust_open(zLockFile,O_RDONLY|O_CREAT|O_EXCL,0600); + if( fd<0 ){ + /* failed to open/create the file, someone else may have stolen the lock */ + int tErrno = errno; + if( EEXIST == tErrno ){ + rc = SQLITE_BUSY; + } else { + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + } + return rc; + } + robust_close(pFile, fd, __LINE__); + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + return rc; +} - /* If some thread using this PID has a lock via a different unixFile* - ** handle that precludes the requested lock, return BUSY. - */ - if( (pFile->eFileLock!=pInode->eFileLock && - (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) - ){ - rc = SQLITE_BUSY; - goto afp_end_lock; - } +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** When the locking level reaches NO_LOCK, delete the lock file. +*/ +static int dotlockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, + pFile->eFileLock, getpid())); + assert( eFileLock<=SHARED_LOCK ); - /* If a SHARED lock is requested, and some thread using this PID already - ** has a SHARED or RESERVED lock, then increment reference counts and - ** return SQLITE_OK. + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* To downgrade to shared, simply update our internal notion of the + ** lock state. No need to mess with the file on disk. */ - if( eFileLock==SHARED_LOCK && - (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ - assert( eFileLock==SHARED_LOCK ); - assert( pFile->eFileLock==0 ); - assert( pInode->nShared>0 ); + if( eFileLock==SHARED_LOCK ){ pFile->eFileLock = SHARED_LOCK; - pInode->nShared++; - pInode->nLock++; - goto afp_end_lock; + return SQLITE_OK; } - - /* A PENDING lock is needed before acquiring a SHARED lock and before - ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will - ** be released. - */ - if( eFileLock==SHARED_LOCK - || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockdbPath, pFile, PENDING_BYTE, 1, 1); - if (failed) { - rc = failed; - goto afp_end_lock; + + /* To fully unlock the database, delete the lock file */ + assert( eFileLock==NO_LOCK ); + if( unlink(zLockFile) ){ + int rc = 0; + int tErrno = errno; + if( ENOENT != tErrno ){ + rc = SQLITE_IOERR_UNLOCK; + } + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; } + return rc; } - - /* If control gets to this point, then actually go ahead and make - ** operating system calls for the specified lock. - */ - if( eFileLock==SHARED_LOCK ){ - int lrc1, lrc2, lrc1Errno; - long lk, mask; - - assert( pInode->nShared==0 ); - assert( pInode->eFileLock==0 ); - - mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; - /* Now get the read-lock SHARED_LOCK */ - /* note that the quality of the randomness doesn't matter that much */ - lk = random(); - pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); - lrc1 = afpSetLock(context->dbPath, pFile, - SHARED_FIRST+pInode->sharedByte, 1, 1); - if( IS_LOCK_ERROR(lrc1) ){ - lrc1Errno = pFile->lastErrno; - } - /* Drop the temporary PENDING lock */ - lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); - - if( IS_LOCK_ERROR(lrc1) ) { - pFile->lastErrno = lrc1Errno; - rc = lrc1; - goto afp_end_lock; - } else if( IS_LOCK_ERROR(lrc2) ){ - rc = lrc2; - goto afp_end_lock; - } else if( lrc1 != SQLITE_OK ) { - rc = lrc1; - } else { - pFile->eFileLock = SHARED_LOCK; - pInode->nLock++; - pInode->nShared = 1; - } - }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ - /* We are trying for an exclusive lock but another thread in this - ** same process is still holding a shared lock. */ - rc = SQLITE_BUSY; - }else{ - /* The request was for a RESERVED or EXCLUSIVE lock. It is - ** assumed that there is a SHARED or greater lock on the file - ** already. - */ - int failed = 0; - assert( 0!=pFile->eFileLock ); - if (eFileLock >= RESERVED_LOCK && pFile->eFileLock < RESERVED_LOCK) { - /* Acquire a RESERVED lock */ - failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); - if( !failed ){ - context->reserved = 1; - } - } - if (!failed && eFileLock == EXCLUSIVE_LOCK) { - /* Acquire an EXCLUSIVE lock */ - - /* Remove the shared lock before trying the range. we'll need to - ** reestablish the shared lock if we can't get the afpUnlock - */ - if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + - pInode->sharedByte, 1, 0)) ){ - int failed2 = SQLITE_OK; - /* now attemmpt to get the exclusive lock range */ - failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, - SHARED_SIZE, 1); - if( failed && (failed2 = afpSetLock(context->dbPath, pFile, - SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ - /* Can't reestablish the shared lock. Sqlite can't deal, this is - ** a critical I/O error - */ - rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : - SQLITE_IOERR_LOCK; - goto afp_end_lock; - } - }else{ - rc = failed; - } - } - if( failed ){ - rc = failed; - } - } - - if( rc==SQLITE_OK ){ - pFile->eFileLock = eFileLock; - pInode->eFileLock = eFileLock; - }else if( eFileLock==EXCLUSIVE_LOCK ){ - pFile->eFileLock = PENDING_LOCK; - pInode->eFileLock = PENDING_LOCK; + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* +** Close a file. Make sure the lock has been released before closing. +*/ +static int dotlockClose(sqlite3_file *id) { + int rc; + if( id ){ + unixFile *pFile = (unixFile*)id; + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); } - -afp_end_lock: - unixLeaveMutex(); - OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), - rc==SQLITE_OK ? "ok" : "failed")); + rc = closeUnixFile(id); return rc; } +/****************** End of the dot-file lock implementation ******************* +******************************************************************************/ -/* -** Lower the locking level on file descriptor pFile to eFileLock. eFileLock -** must be either NO_LOCK or SHARED_LOCK. +/****************************************************************************** +************************** Begin flock Locking ******************************** ** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. +** Use the flock() system call to do file locking. +** +** flock() locking is like dot-file locking in that the various +** fine-grain locking levels supported by SQLite are collapsed into +** a single exclusive lock. In other words, SHARED, RESERVED, and +** PENDING locks are the same thing as an EXCLUSIVE lock. SQLite +** still works when you do this, but concurrency is reduced since +** only a single process can be reading the database at a time. +** +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off or if +** compiling for VXWORKS. */ -static int afpUnlock(sqlite3_file *id, int eFileLock) { - int rc = SQLITE_OK; - unixFile *pFile = (unixFile*)id; - unixInodeInfo *pInode; - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - int skipShared = 0; -#ifdef SQLITE_TEST - int h = pFile->h; +#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS + +/* +** Retry flock() calls that fail with EINTR +*/ +#ifdef EINTR +static int robust_flock(int fd, int op){ + int rc; + do{ rc = flock(fd,op); }while( rc<0 && errno==EINTR ); + return rc; +} +#else +# define robust_flock(a,b) flock(a,b) #endif + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + assert( pFile ); - OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, - pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, - getpid())); - - assert( eFileLock<=SHARED_LOCK ); - if( pFile->eFileLock<=eFileLock ){ - return SQLITE_OK; - } - unixEnterMutex(); - pInode = pFile->pInode; - assert( pInode->nShared!=0 ); + + /* Check if a thread in this process holds such a lock */ if( pFile->eFileLock>SHARED_LOCK ){ - assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); - -#ifndef NDEBUG - /* When reducing a lock such that other processes can start - ** reading the database file again, make sure that the - ** transaction counter was updated if any part of the database - ** file changed. If the transaction counter is not updated, - ** other connections to the same file might not realize that - ** the file has changed and hence might not know to flush their - ** cache. The use of a stale cache can lead to database corruption. - */ - assert( pFile->inNormalWrite==0 - || pFile->dbUpdate==0 - || pFile->transCntrChng==1 ); - pFile->inNormalWrite = 0; -#endif - - if( pFile->eFileLock==EXCLUSIVE_LOCK ){ - rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); - if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ - /* only re-establish the shared lock if necessary */ - int sharedLockByte = SHARED_FIRST+pInode->sharedByte; - rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); - } else { - skipShared = 1; + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + /* attempt to get the lock */ + int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); + if( !lrc ){ + /* got the lock, unlock it */ + lrc = robust_flock(pFile->h, LOCK_UN); + if ( lrc ) { + int tErrno = errno; + /* unlock failed with an error */ + lrc = SQLITE_IOERR_UNLOCK; + if( IS_LOCK_ERROR(lrc) ){ + pFile->lastErrno = tErrno; + rc = lrc; + } } - } - if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ - rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); - } - if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ - rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); - if( !rc ){ - context->reserved = 0; + } else { + int tErrno = errno; + reserved = 1; + /* someone else might have it reserved */ + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(lrc) ){ + pFile->lastErrno = tErrno; + rc = lrc; } } - if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1)){ - pInode->eFileLock = SHARED_LOCK; - } } - if( rc==SQLITE_OK && eFileLock==NO_LOCK ){ + OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); - /* Decrement the shared lock counter. Release the lock using an - ** OS call only when all threads in this same process have released - ** the lock. - */ - unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; - pInode->nShared--; - if( pInode->nShared==0 ){ - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); - if( !skipShared ){ - rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); - } - if( !rc ){ - pInode->eFileLock = NO_LOCK; - pFile->eFileLock = NO_LOCK; - } - } - if( rc==SQLITE_OK ){ - pInode->nLock--; - assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - rc = closePendingFds(pFile); - } - } +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + rc = SQLITE_OK; + reserved=1; } - - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + *pResOut = reserved; return rc; } /* -** Close a file & cleanup AFP specific locking context +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** flock() only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. */ -static int afpClose(sqlite3_file *id) { +static int flockLock(sqlite3_file *id, int eFileLock) { int rc = SQLITE_OK; - if( id ){ - unixFile *pFile = (unixFile*)id; - afpUnlock(id, NO_LOCK); - unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* grab an exclusive lock */ + + if (robust_flock(pFile->h, LOCK_EX | LOCK_NB)) { + int tErrno = errno; + /* didn't get, must be busy */ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; } - releaseInodeInfo(pFile); - sqlite3_free(pFile->lockingContext); - rc = closeUnixFile(id); - unixLeaveMutex(); + } else { + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + } + OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + rc = SQLITE_BUSY; } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ return rc; } -#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + /* -** The code above is the AFP lock implementation. The code is specific -** to MacOSX and does not work on other unix platforms. No alternative -** is available. If you don't compile for a mac, then the "unix-afp" -** VFS is not available. +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. ** -********************* End of the AFP lock implementation ********************** -******************************************************************************/ - -/****************************************************************************** -*************************** Begin NFS Locking ********************************/ +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int flockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, + pFile->eFileLock, getpid())); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really, unlock. */ + if( robust_flock(pFile->h, LOCK_UN) ){ +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + return SQLITE_OK; +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return SQLITE_IOERR_UNLOCK; + }else{ + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; + } +} -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE /* - ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock - ** must be either NO_LOCK or SHARED_LOCK. - ** - ** If the locking level of the file descriptor is already at or below - ** the requested locking level, this routine is a no-op. - */ -static int nfsUnlock(sqlite3_file *id, int eFileLock){ - return _posixUnlock(id, eFileLock, 1); +** Close a file. +*/ +static int flockClose(sqlite3_file *id) { + if( id ){ + flockUnlock(id, NO_LOCK); + } + return closeUnixFile(id); } -#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ -/* -** The code above is the NFS lock implementation. The code is specific -** to MacOSX and does not work on other unix platforms. No alternative -** is available. -** -********************* End of the NFS lock implementation ********************** +#endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ + +/******************* End of the flock lock implementation ********************* ******************************************************************************/ /****************************************************************************** -**************** Non-locking sqlite3_file methods ***************************** +************************ Begin Named Semaphore Locking ************************ ** -** The next division contains implementations for all methods of the -** sqlite3_file object other than the locking methods. The locking -** methods were defined in divisions above (one locking method per -** division). Those methods that are common to all locking modes -** are gather together into this division. +** Named semaphore locking is only supported on VxWorks. +** +** Semaphore locking is like dot-lock and flock in that it really only +** supports EXCLUSIVE locking. Only a single process can read or write +** the database file at a time. This reduces potential concurrency, but +** makes the lock implementation much easier. */ +#if OS_VXWORKS /* -** Seek to the offset passed as the second argument, then read cnt -** bytes into pBuf. Return the number of bytes actually read. -** -** NB: If you define USE_PREAD or USE_PREAD64, then it might also -** be necessary to define _XOPEN_SOURCE to be 500. This varies from -** one system to another. Since SQLite does not define USE_PREAD -** any any form by default, we will not attempt to define _XOPEN_SOURCE. -** See tickets #2741 and #2681. -** -** To avoid stomping the errno value on a failed read the lastErrno value -** is set before returning. +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif - TIMER_START; -#if defined(USE_PREAD) - got = pread(id->h, pBuf, cnt, offset); - SimulateIOError( got = -1 ); -#elif defined(USE_PREAD64) - got = pread64(id->h, pBuf, cnt, offset); - SimulateIOError( got = -1 ); -#else - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; +static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + sem_t *pSem = pFile->pInode->pSem; + struct stat statBuf; + + if( sem_trywait(pSem)==-1 ){ + int tErrno = errno; + if( EAGAIN != tErrno ){ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + pFile->lastErrno = tErrno; + } else { + /* someone else has the lock when we are in NO_LOCK */ + reserved = (pFile->eFileLock < SHARED_LOCK); + } }else{ - ((unixFile*)id)->lastErrno = 0; + /* we could have it if we want it */ + sem_post(pSem); } - return -1; } - got = read(id->h, pBuf, cnt); -#endif - TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } - OSTRACE(("READ %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + OSTRACE(("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; } /* -** Read data from a file into a buffer. Return SQLITE_OK if all -** bytes were read successfully and SQLITE_IOERR if anything goes -** wrong. -*/ -static int unixRead( - sqlite3_file *id, - void *pBuf, - int amt, - sqlite3_int64 offset -){ - unixFile *pFile = (unixFile *)id; - int got; - assert( id ); - - /* If this is a database file (not a journal, master-journal or temp - ** file), the bytes in the locking range should never be read or written. */ -#if 0 - assert( pFile->pUnused==0 - || offset>=PENDING_BYTE+512 - || offset+amt<=PENDING_BYTE - ); -#endif - - got = seekAndRead(pFile, offset, pBuf, amt); - if( got==amt ){ - return SQLITE_OK; - }else if( got<0 ){ - /* lastErrno set by seekAndRead */ - return SQLITE_IOERR_READ; - }else{ - pFile->lastErrno = 0; /* not a system error */ - /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pBuf)[got], 0, amt-got); - return SQLITE_IOERR_SHORT_READ; - } -} - -/* -** Seek to the offset in id->offset then read cnt bytes into pBuf. -** Return the number of bytes actually read. Update the offset. +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: ** -** To avoid stomping the errno value on a failed write the lastErrno value -** is set before returning. +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** Semaphore locks only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. */ -static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif - TIMER_START; -#if defined(USE_PREAD) - got = pwrite(id->h, pBuf, cnt, offset); -#elif defined(USE_PREAD64) - got = pwrite64(id->h, pBuf, cnt, offset); -#else - newOffset = lseek(id->h, offset, SEEK_SET); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } - return -1; +static int semLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int fd; + sem_t *pSem = pFile->pInode->pSem; + int rc = SQLITE_OK; + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + rc = SQLITE_OK; + goto sem_end_lock; } - got = write(id->h, pBuf, cnt); -#endif - TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; + + /* lock semaphore now but bail out when already locked. */ + if( sem_trywait(pSem)==-1 ){ + rc = SQLITE_BUSY; + goto sem_end_lock; } - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; -} + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + sem_end_lock: + return rc; +} /* -** Write data from a buffer into a file. Return SQLITE_OK on success -** or some other error code on failure. +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. */ -static int unixWrite( - sqlite3_file *id, - const void *pBuf, - int amt, - sqlite3_int64 offset -){ +static int semUnlock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - int wrote = 0; - assert( id ); - assert( amt>0 ); - - /* If this is a database file (not a journal, master-journal or temp - ** file), the bytes in the locking range should never be read or written. */ -#if 0 - assert( pFile->pUnused==0 - || offset>=PENDING_BYTE+512 - || offset+amt<=PENDING_BYTE - ); -#endif + sem_t *pSem = pFile->pInode->pSem; -#ifndef NDEBUG - /* If we are doing a normal write to a database file (as opposed to - ** doing a hot-journal rollback or a write to some file other than a - ** normal database file) then record the fact that the database - ** has changed. If the transaction counter is modified, record that - ** fact too. - */ - if( pFile->inNormalWrite ){ - pFile->dbUpdate = 1; /* The database has been modified */ - if( offset<=24 && offset+amt>=27 ){ - int rc; - char oldCntr[4]; - SimulateIOErrorBenign(1); - rc = seekAndRead(pFile, 24, oldCntr, 4); - SimulateIOErrorBenign(0); - if( rc!=4 || memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){ - pFile->transCntrChng = 1; /* The transaction counter has changed */ - } - } + assert( pFile ); + assert( pSem ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, + pFile->eFileLock, getpid())); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; } -#endif - - while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ - amt -= wrote; - offset += wrote; - pBuf = &((char*)pBuf)[wrote]; + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; } - SimulateIOError(( wrote=(-1), amt=1 )); - SimulateDiskfullError(( wrote=0, amt=1 )); - - if( amt>0 ){ - if( wrote<0 ){ - /* lastErrno set by seekAndWrite */ - return SQLITE_IOERR_WRITE; - }else{ - pFile->lastErrno = 0; /* not a system error */ - return SQLITE_FULL; + + /* no, really unlock. */ + if ( sem_post(pSem)==-1 ) { + int rc, tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; } + return rc; } - + pFile->eFileLock = NO_LOCK; return SQLITE_OK; } -#ifdef SQLITE_TEST -/* -** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occurring at the right times. -*/ -SQLITE_API int sqlite3_sync_count = 0; -SQLITE_API int sqlite3_fullsync_count = 0; -#endif - /* -** We do not trust systems to provide a working fdatasync(). Some do. -** Others do no. To be safe, we will stick with the (slower) fsync(). -** If you know that your system does support fdatasync() correctly, -** then simply compile with -Dfdatasync=fdatasync -*/ -#if !defined(fdatasync) && !defined(__linux__) -# define fdatasync fsync -#endif + ** Close a file. + */ +static int semClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + semUnlock(id, NO_LOCK); + assert( pFile ); + unixEnterMutex(); + releaseInodeInfo(pFile); + unixLeaveMutex(); + closeUnixFile(id); + } + return SQLITE_OK; +} +#endif /* OS_VXWORKS */ /* -** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not -** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently -** only available on Mac OS X. But that could change. -*/ -#ifdef F_FULLFSYNC -# define HAVE_FULLFSYNC 1 -#else -# define HAVE_FULLFSYNC 0 -#endif +** Named semaphore locking is only available on VxWorks. +** +*************** End of the named semaphore lock implementation **************** +******************************************************************************/ -/* -** The fsync() system call does not work as advertised on many -** unix systems. The following procedure is an attempt to make -** it work better. +/****************************************************************************** +*************************** Begin AFP Locking ********************************* ** -** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful -** for testing when we want to run through the test suite quickly. -** You are strongly advised *not* to deploy with SQLITE_NO_SYNC -** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash -** or power failure will likely corrupt the database file. +** AFP is the Apple Filing Protocol. AFP is a network filesystem found +** on Apple Macintosh computers - both OS9 and OSX. ** -** SQLite sets the dataOnly flag if the size of the file is unchanged. -** The idea behind dataOnly is that it should only write the file content -** to disk, not the inode. We only set dataOnly if the file size is -** unchanged since the file size is part of the inode. However, -** Ted Ts'o tells us that fdatasync() will also write the inode if the -** file size has changed. The only real difference between fdatasync() -** and fsync(), Ted tells us, is that fdatasync() will not flush the -** inode if the mtime or owner or other inode attributes have changed. -** We only care about the file size, not the other file attributes, so -** as far as SQLite is concerned, an fdatasync() is always adequate. -** So, we always use fdatasync() if it is available, regardless of -** the value of the dataOnly flag. +** Third-party implementations of AFP are available. But this code here +** only works on OSX. */ -static int full_fsync(int fd, int fullSync, int dataOnly){ - int rc; - - /* The following "ifdef/elif/else/" block has the same structure as - ** the one below. It is replicated here solely to avoid cluttering - ** up the real code with the UNUSED_PARAMETER() macros. - */ -#ifdef SQLITE_NO_SYNC - UNUSED_PARAMETER(fd); - UNUSED_PARAMETER(fullSync); - UNUSED_PARAMETER(dataOnly); -#elif HAVE_FULLFSYNC - UNUSED_PARAMETER(dataOnly); -#else - UNUSED_PARAMETER(fullSync); - UNUSED_PARAMETER(dataOnly); -#endif - /* Record the number of times that we do a normal fsync() and - ** FULLSYNC. This is used during testing to verify that this procedure - ** gets called with the correct arguments. - */ -#ifdef SQLITE_TEST - if( fullSync ) sqlite3_fullsync_count++; - sqlite3_sync_count++; -#endif +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** The afpLockingContext structure contains all afp lock specific state +*/ +typedef struct afpLockingContext afpLockingContext; +struct afpLockingContext { + int reserved; + const char *dbPath; /* Name of the open file */ +}; - /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a - ** no-op - */ -#ifdef SQLITE_NO_SYNC - rc = SQLITE_OK; -#elif HAVE_FULLFSYNC - if( fullSync ){ - rc = fcntl(fd, F_FULLFSYNC, 0); - }else{ - rc = 1; - } - /* If the FULLFSYNC failed, fall back to attempting an fsync(). - ** It shouldn't be possible for fullfsync to fail on the local - ** file system (on OSX), so failure indicates that FULLFSYNC - ** isn't supported for this file system. So, attempt an fsync - ** and (for now) ignore the overhead of a superfluous fcntl call. - ** It'd be better to detect fullfsync support once and avoid - ** the fcntl call every time sync is called. - */ - if( rc ) rc = fsync(fd); +struct ByteRangeLockPB2 +{ + unsigned long long offset; /* offset to first byte to lock */ + unsigned long long length; /* nbr of bytes to lock */ + unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ + unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ + unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ + int fd; /* file desc to assoc this lock with */ +}; -#elif defined(__APPLE__) - /* fdatasync() on HFS+ doesn't yet flush the file size if it changed correctly - ** so currently we default to the macro that redefines fdatasync to fsync - */ - rc = fsync(fd); -#else - rc = fdatasync(fd); -#if OS_VXWORKS - if( rc==-1 && errno==ENOTSUP ){ - rc = fsync(fd); - } -#endif /* OS_VXWORKS */ -#endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */ +#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) - if( OS_VXWORKS && rc!= -1 ){ - rc = 0; +/* +** This is a utility for setting or clearing a bit-range lock on an +** AFP filesystem. +** +** Return SQLITE_OK on success, SQLITE_BUSY on failure. +*/ +static int afpSetLock( + const char *path, /* Name of the file to be locked or unlocked */ + unixFile *pFile, /* Open file descriptor on path */ + unsigned long long offset, /* First byte to be locked */ + unsigned long long length, /* Number of bytes to lock */ + int setLockFlag /* True to set lock. False to clear lock */ +){ + struct ByteRangeLockPB2 pb; + int err; + + pb.unLockFlag = setLockFlag ? 0 : 1; + pb.startEndFlag = 0; + pb.offset = offset; + pb.length = length; + pb.fd = pFile->h; + + OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), + offset, length)); + err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); + if ( err==-1 ) { + int rc; + int tErrno = errno; + OSTRACE(("AFPSETLOCK failed to fsctl() '%s' %d %s\n", + path, tErrno, strerror(tErrno))); +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + rc = SQLITE_BUSY; +#else + rc = sqliteErrorFromPosixError(tErrno, + setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); +#endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + return rc; + } else { + return SQLITE_OK; } - return rc; } /* -** Make sure all writes to a particular file are committed to disk. -** -** If dataOnly==0 then both the file itself and its metadata (file -** size, access time, etc) are synced. If dataOnly!=0 then only the -** file data is synced. -** -** Under Unix, also make sure that the directory entry for the file -** has been created by fsync-ing the directory that contains the file. -** If we do not do this and we encounter a power failure, the directory -** entry for the journal might not exist after we reboot. The next -** SQLite to access the file will not know that the journal exists (because -** the directory entry for the journal was never created) and the transaction -** will not roll back - possibly leading to database corruption. +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static int unixSync(sqlite3_file *id, int flags){ - int rc; +static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; unixFile *pFile = (unixFile*)id; - - int isDataOnly = (flags&SQLITE_SYNC_DATAONLY); - int isFullsync = (flags&0x0F)==SQLITE_SYNC_FULL; - - /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ - assert((flags&0x0F)==SQLITE_SYNC_NORMAL - || (flags&0x0F)==SQLITE_SYNC_FULL - ); - - /* Unix cannot, but some systems may return SQLITE_FULL from here. This - ** line is to test that doing so does not cause any problems. - */ - SimulateDiskfullError( return SQLITE_FULL ); - + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + assert( pFile ); - OSTRACE(("SYNC %-3d\n", pFile->h)); - rc = full_fsync(pFile->h, isFullsync, isDataOnly); - SimulateIOError( rc=1 ); - if( rc ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_FSYNC; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + if( context->reserved ){ + *pResOut = 1; + return SQLITE_OK; } - if( pFile->dirfd>=0 ){ - int err; - OSTRACE(("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd, - HAVE_FULLFSYNC, isFullsync)); -#ifndef SQLITE_DISABLE_DIRSYNC - /* The directory sync is only attempted if full_fsync is - ** turned off or unavailable. If a full_fsync occurred above, - ** then the directory sync is superfluous. - */ - if( (!HAVE_FULLFSYNC || !isFullsync) && full_fsync(pFile->dirfd,0,0) ){ - /* - ** We have received multiple reports of fsync() returning - ** errors when applied to directories on certain file systems. - ** A failed directory sync is not a big deal. So it seems - ** better to ignore the error. Ticket #1657 - */ - /* pFile->lastErrno = errno; */ - /* return SQLITE_IOERR; */ + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ + if( !reserved ){ + /* lock the RESERVED byte */ + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( SQLITE_OK==lrc ){ + /* if we succeeded in taking the reserved lock, unlock it to restore + ** the original state */ + lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + } else { + /* if we failed to get the lock then someone else must have it */ + reserved = 1; } -#endif - err = close(pFile->dirfd); /* Only need to sync once, so close the */ - if( err==0 ){ /* directory when we are done */ - pFile->dirfd = -1; - }else{ - pFile->lastErrno = errno; - rc = SQLITE_IOERR_DIR_CLOSE; + if( IS_LOCK_ERROR(lrc) ){ + rc=lrc; } } + + unixLeaveMutex(); + OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; return rc; } /* -** Truncate an open file to a specified size +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. */ -static int unixTruncate(sqlite3_file *id, i64 nByte){ - unixFile *pFile = (unixFile *)id; - int rc; +static int afpLock(sqlite3_file *id, int eFileLock){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode = pFile->pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + assert( pFile ); - SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pInode->eFileLock), pInode->nShared , getpid())); - /* If the user has configured a chunk-size for this file, truncate the - ** file so that it consists of an integer number of chunks (i.e. the - ** actual file size after the operation may be larger than the requested - ** size). + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. */ - if( pFile->szChunk ){ - nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; - } + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (afp)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } - rc = ftruncate(pFile->h, (off_t)nByte); - if( rc ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_TRUNCATE; + /* Make sure the locking sequence is correct + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + unixEnterMutex(); + pInode = pFile->pInode; + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto afp_end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto afp_end_lock; + } + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockdbPath, pFile, PENDING_BYTE, 1, 1); + if (failed) { + rc = failed; + goto afp_end_lock; + } + } + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( eFileLock==SHARED_LOCK ){ + int lrc1, lrc2, lrc1Errno; + long lk, mask; + + assert( pInode->nShared==0 ); + assert( pInode->eFileLock==0 ); + + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; + /* Now get the read-lock SHARED_LOCK */ + /* note that the quality of the randomness doesn't matter that much */ + lk = random(); + pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); + lrc1 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST+pInode->sharedByte, 1, 1); + if( IS_LOCK_ERROR(lrc1) ){ + lrc1Errno = pFile->lastErrno; + } + /* Drop the temporary PENDING lock */ + lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + + if( IS_LOCK_ERROR(lrc1) ) { + pFile->lastErrno = lrc1Errno; + rc = lrc1; + goto afp_end_lock; + } else if( IS_LOCK_ERROR(lrc2) ){ + rc = lrc2; + goto afp_end_lock; + } else if( lrc1 != SQLITE_OK ) { + rc = lrc1; + } else { + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; }else{ -#ifndef NDEBUG - /* If we are doing a normal write to a database file (as opposed to - ** doing a hot-journal rollback or a write to some file other than a - ** normal database file) and we truncate the file to zero length, - ** that effectively updates the change counter. This might happen - ** when restoring a database using the backup API from a zero-length - ** source. + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. */ - if( pFile->inNormalWrite && nByte==0 ){ - pFile->transCntrChng = 1; + int failed = 0; + assert( 0!=pFile->eFileLock ); + if (eFileLock >= RESERVED_LOCK && pFile->eFileLock < RESERVED_LOCK) { + /* Acquire a RESERVED lock */ + failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( !failed ){ + context->reserved = 1; + } + } + if (!failed && eFileLock == EXCLUSIVE_LOCK) { + /* Acquire an EXCLUSIVE lock */ + + /* Remove the shared lock before trying the range. we'll need to + ** reestablish the shared lock if we can't get the afpUnlock + */ + if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + + pInode->sharedByte, 1, 0)) ){ + int failed2 = SQLITE_OK; + /* now attemmpt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + SHARED_SIZE, 1); + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ + /* Can't reestablish the shared lock. Sqlite can't deal, this is + ** a critical I/O error + */ + rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : + SQLITE_IOERR_LOCK; + goto afp_end_lock; + } + }else{ + rc = failed; + } + } + if( failed ){ + rc = failed; } -#endif - - return SQLITE_OK; } -} - -/* -** Determine the current size of a file in bytes -*/ -static int unixFileSize(sqlite3_file *id, i64 *pSize){ - int rc; - struct stat buf; - assert( id ); - rc = fstat(((unixFile*)id)->h, &buf); - SimulateIOError( rc=1 ); - if( rc!=0 ){ - ((unixFile*)id)->lastErrno = errno; - return SQLITE_IOERR_FSTAT; + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; } - *pSize = buf.st_size; - - /* When opening a zero-size database, the findInodeInfo() procedure - ** writes a single byte into that file in order to work around a bug - ** in the OS-X msdos filesystem. In order to avoid problems with upper - ** layers, we need to report this file size as zero even though it is - ** really 1. Ticket #3260. - */ - if( *pSize==1 ) *pSize = 0; - - - return SQLITE_OK; + +afp_end_lock: + unixLeaveMutex(); + OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; } -#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) /* -** Handler for proxy-locking file-control verbs. Defined below in the -** proxying locking division. +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. */ -static int proxyFileControl(sqlite3_file*,int,void*); +static int afpUnlock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + int skipShared = 0; +#ifdef SQLITE_TEST + int h = pFile->h; #endif -/* -** This function is called to handle the SQLITE_FCNTL_SIZE_HINT -** file-control operation. -** -** If the user has configured a chunk-size for this file, it could be -** that the file needs to be extended at this point. Otherwise, the -** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. -*/ -static int fcntlSizeHint(unixFile *pFile, i64 nByte){ - if( pFile->szChunk ){ - i64 nSize; /* Required file size */ - struct stat buf; /* Used to hold return values of fstat() */ - - if( fstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + getpid())); - nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; - if( nSize>(i64)buf.st_size ){ -#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE - if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ - return SQLITE_IOERR_WRITE; + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + unixEnterMutex(); + pInode = pFile->pInode; + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifndef NDEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); + pFile->inNormalWrite = 0; +#endif + + if( pFile->eFileLock==EXCLUSIVE_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ + /* only re-establish the shared lock if necessary */ + int sharedLockByte = SHARED_FIRST+pInode->sharedByte; + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); + } else { + skipShared = 1; } -#else - /* If the OS does not have posix_fallocate(), fake it. First use - ** ftruncate() to set the file size, then write a single byte to - ** the last byte in each block within the extended region. This - ** is the same technique used by glibc to implement posix_fallocate() - ** on systems that do not have a real fallocate() system call. - */ - int nBlk = buf.st_blksize; /* File-system block size */ - i64 iWrite; /* Next offset to write to */ - int nWrite; /* Return value from seekAndWrite() */ - - if( ftruncate(pFile->h, nSize) ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_TRUNCATE; + } + if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + } + if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ + rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + if( !rc ){ + context->reserved = 0; } - iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; - do { - nWrite = seekAndWrite(pFile, iWrite, "", 1); - iWrite += nBlk; - } while( nWrite==1 && iWritenShared>1)){ + pInode->eFileLock = SHARED_LOCK; } } + if( rc==SQLITE_OK && eFileLock==NO_LOCK ){ - return SQLITE_OK; -} - -/* -** Information and control of an open file handle. -*/ -static int unixFileControl(sqlite3_file *id, int op, void *pArg){ - switch( op ){ - case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((unixFile*)id)->eFileLock; - return SQLITE_OK; - } - case SQLITE_LAST_ERRNO: { - *(int*)pArg = ((unixFile*)id)->lastErrno; - return SQLITE_OK; - } - case SQLITE_FCNTL_CHUNK_SIZE: { - ((unixFile*)id)->szChunk = *(int *)pArg; - return SQLITE_OK; - } - case SQLITE_FCNTL_SIZE_HINT: { - return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); - } -#ifndef NDEBUG - /* The pager calls this method to signal that it has done - ** a rollback and that the database is therefore unchanged and - ** it hence it is OK for the transaction change counter to be - ** unchanged. + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. */ - case SQLITE_FCNTL_DB_UNCHANGED: { - ((unixFile*)id)->dbUpdate = 0; - return SQLITE_OK; + unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; + pInode->nShared--; + if( pInode->nShared==0 ){ + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( !skipShared ){ + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); + } + if( !rc ){ + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } } -#endif -#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - case SQLITE_SET_LOCKPROXYFILE: - case SQLITE_GET_LOCKPROXYFILE: { - return proxyFileControl(id,op,pArg); + if( rc==SQLITE_OK ){ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ){ + closePendingFds(pFile); + } } -#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ } - return SQLITE_ERROR; + + unixLeaveMutex(); + if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + return rc; } /* -** Return the sector size in bytes of the underlying block device for -** the specified file. This is almost always 512 bytes, but may be -** larger for some devices. -** -** SQLite code assumes this function cannot fail. It also assumes that -** if two files are created in the same file-system directory (i.e. -** a database and its journal file) that the sector size will be the -** same for both. +** Close a file & cleanup AFP specific locking context */ -static int unixSectorSize(sqlite3_file *NotUsed){ - UNUSED_PARAMETER(NotUsed); - return SQLITE_DEFAULT_SECTOR_SIZE; +static int afpClose(sqlite3_file *id) { + int rc = SQLITE_OK; + if( id ){ + unixFile *pFile = (unixFile*)id; + afpUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pInode && pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); + } + return rc; } +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ /* -** Return the device characteristics for the file. This is always 0 for unix. -*/ -static int unixDeviceCharacteristics(sqlite3_file *NotUsed){ - UNUSED_PARAMETER(NotUsed); - return 0; -} +** The code above is the AFP lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. If you don't compile for a mac, then the "unix-afp" +** VFS is not available. +** +********************* End of the AFP lock implementation ********************** +******************************************************************************/ -#ifndef SQLITE_OMIT_WAL +/****************************************************************************** +*************************** Begin NFS Locking ********************************/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* + ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock + ** must be either NO_LOCK or SHARED_LOCK. + ** + ** If the locking level of the file descriptor is already at or below + ** the requested locking level, this routine is a no-op. + */ +static int nfsUnlock(sqlite3_file *id, int eFileLock){ + return posixUnlock(id, eFileLock, 1); +} +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ /* -** Object used to represent an shared memory buffer. -** -** When multiple threads all reference the same wal-index, each thread -** has its own unixShm object, but they all point to a single instance -** of this unixShmNode object. In other words, each wal-index is opened -** only once per process. -** -** Each unixShmNode object is connected to a single unixInodeInfo object. -** We could coalesce this object into unixInodeInfo, but that would mean -** every open file that does not use shared memory (in other words, most -** open files) would have to carry around this extra information. So -** the unixInodeInfo object contains a pointer to this unixShmNode object -** and the unixShmNode object is created only when needed. -** -** unixMutexHeld() must be true when creating or destroying -** this object or while reading or writing the following fields: -** -** nRef +** The code above is the NFS lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. ** -** The following fields are read-only after the object is created: -** -** fid -** zFilename +********************* End of the NFS lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +**************** Non-locking sqlite3_file methods ***************************** ** -** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and -** unixMutexHeld() is true when reading or writing any other field -** in this structure. +** The next division contains implementations for all methods of the +** sqlite3_file object other than the locking methods. The locking +** methods were defined in divisions above (one locking method per +** division). Those methods that are common to all locking modes +** are gather together into this division. */ -struct unixShmNode { - unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ - sqlite3_mutex *mutex; /* Mutex to access this object */ - char *zFilename; /* Name of the mmapped file */ - int h; /* Open file descriptor */ - int szRegion; /* Size of shared-memory regions */ - int nRegion; /* Size of array apRegion */ - char **apRegion; /* Array of mapped shared-memory regions */ - int nRef; /* Number of unixShm objects pointing to this */ - unixShm *pFirst; /* All unixShm objects pointing to this */ -#ifdef SQLITE_DEBUG - u8 exclMask; /* Mask of exclusive locks held */ - u8 sharedMask; /* Mask of shared locks held */ - u8 nextShmId; /* Next available unixShm.id value */ -#endif -}; /* -** Structure used internally by this VFS to record the state of an -** open shared memory connection. -** -** The following fields are initialized when this object is created and -** are read-only thereafter: +** Seek to the offset passed as the second argument, then read cnt +** bytes into pBuf. Return the number of bytes actually read. ** -** unixShm.pFile -** unixShm.id +** NB: If you define USE_PREAD or USE_PREAD64, then it might also +** be necessary to define _XOPEN_SOURCE to be 500. This varies from +** one system to another. Since SQLite does not define USE_PREAD +** any any form by default, we will not attempt to define _XOPEN_SOURCE. +** See tickets #2741 and #2681. ** -** All other fields are read/write. The unixShm.pFile->mutex must be held -** while accessing any read/write fields. +** To avoid stomping the errno value on a failed read the lastErrno value +** is set before returning. */ -struct unixShm { - unixShmNode *pShmNode; /* The underlying unixShmNode object */ - unixShm *pNext; /* Next unixShm with the same unixShmNode */ - u8 hasMutex; /* True if holding the unixShmNode mutex */ - u16 sharedMask; /* Mask of shared locks held */ - u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG - u8 id; /* Id of this connection within its unixShmNode */ +static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ + int got; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) + i64 newOffset; #endif -}; - -/* -** Constants used for locking -*/ -#define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ -#define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ - -/* -** Apply posix advisory locks for all bytes from ofst through ofst+n-1. -** -** Locks block if the mask is exactly UNIX_SHM_C and are non-blocking -** otherwise. -*/ -static int unixShmSystemLock( - unixShmNode *pShmNode, /* Apply locks to this open shared-memory segment */ - int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ - int ofst, /* First byte of the locking range */ - int n /* Number of bytes to lock */ -){ - struct flock f; /* The posix advisory locking structure */ - int rc = SQLITE_OK; /* Result code form fcntl() */ - - /* Access to the unixShmNode object is serialized by the caller */ - assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); - - /* Shared locks never span more than one byte */ - assert( n==1 || lockType!=F_RDLCK ); - - /* Locks are within range */ - assert( n>=1 && nh, F_SETLK, &f); - rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; - - /* Update the global lock state and do debug tracing */ -#ifdef SQLITE_DEBUG - { u16 mask; - OSTRACE(("SHM-LOCK ")); - mask = (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; - }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; - } - }else{ - if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); - }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); + TIMER_START; +#if defined(USE_PREAD) + do{ got = osPread(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); + SimulateIOError( got = -1 ); +#elif defined(USE_PREAD64) + do{ got = osPread64(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR); + SimulateIOError( got = -1 ); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset-- ); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + ((unixFile*)id)->lastErrno = errno; }else{ - assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); + ((unixFile*)id)->lastErrno = 0; } + return -1; } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); + do{ got = osRead(id->h, pBuf, cnt); }while( got<0 && errno==EINTR ); +#endif + TIMER_END; + if( got<0 ){ + ((unixFile*)id)->lastErrno = errno; } + OSTRACE(("READ %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); + return got; +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int unixRead( + sqlite3_file *id, + void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile *)id; + int got; + assert( id ); + + /* If this is a database file (not a journal, master-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); #endif - return rc; + got = seekAndRead(pFile, offset, pBuf, amt); + if( got==amt ){ + return SQLITE_OK; + }else if( got<0 ){ + /* lastErrno set by seekAndRead */ + return SQLITE_IOERR_READ; + }else{ + pFile->lastErrno = 0; /* not a system error */ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[got], 0, amt-got); + return SQLITE_IOERR_SHORT_READ; + } } - /* -** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0. +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. ** -** This is not a VFS shared-memory method; it is a utility function called -** by VFS shared-memory methods. +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. */ -static void unixShmPurge(unixFile *pFd){ - unixShmNode *p = pFd->pInode->pShmNode; - assert( unixMutexHeld() ); - if( p && p->nRef==0 ){ - int i; - assert( p->pInode==pFd->pInode ); - if( p->mutex ) sqlite3_mutex_free(p->mutex); - for(i=0; inRegion; i++){ - munmap(p->apRegion[i], p->szRegion); +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + int got; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) + i64 newOffset; +#endif + TIMER_START; +#if defined(USE_PREAD) + do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset-- ); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + ((unixFile*)id)->lastErrno = errno; + }else{ + ((unixFile*)id)->lastErrno = 0; } - sqlite3_free(p->apRegion); - if( p->h>=0 ) close(p->h); - p->pInode->pShmNode = 0; - sqlite3_free(p); + return -1; + } + do{ got = osWrite(id->h, pBuf, cnt); }while( got<0 && errno==EINTR ); +#endif + TIMER_END; + if( got<0 ){ + ((unixFile*)id)->lastErrno = errno; } + + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); + return got; } + /* -** Open a shared-memory area associated with open database file pDbFd. -** This particular implementation uses mmapped files. -** -** The file used to implement shared-memory is in the same directory -** as the open database file and has the same name as the open database -** file with the "-shm" suffix added. For example, if the database file -** is "/home/user1/config.db" then the file that is created and mmapped -** for shared memory will be called "/home/user1/config.db-shm". -** -** Another approach to is to use files in /dev/shm or /dev/tmp or an -** some other tmpfs mount. But if a file in a different directory -** from the database file is used, then differing access permissions -** or a chroot() might cause two different processes on the same -** database to end up using different files for shared memory - -** meaning that their memory would not really be shared - resulting -** in database corruption. Nevertheless, this tmpfs file usage -** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" -** or the equivalent. The use of the SQLITE_SHM_DIRECTORY compile-time -** option results in an incompatible build of SQLite; builds of SQLite -** that with differing SQLITE_SHM_DIRECTORY settings attempt to use the -** same database file at the same time, database corruption will likely -** result. The SQLITE_SHM_DIRECTORY compile-time option is considered -** "unsupported" and may go away in a future SQLite release. -** -** When opening a new shared-memory file, if no other instances of that -** file are currently open, in this process or in other processes, then -** the file must be truncated to zero length or have its header cleared. +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. */ -static int unixOpenSharedMemory(unixFile *pDbFd){ - struct unixShm *p = 0; /* The connection to be opened */ - struct unixShmNode *pShmNode; /* The underlying mmapped file */ - int rc; /* Result code */ - unixInodeInfo *pInode; /* The inode of fd */ - char *zShmFilename; /* Name of the file used for SHM */ - int nShmFilename; /* Size of the SHM filename in bytes */ +static int unixWrite( + sqlite3_file *id, + const void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile*)id; + int wrote = 0; + assert( id ); + assert( amt>0 ); - /* Allocate space for the new unixShm object. */ - p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM; - memset(p, 0, sizeof(*p)); - assert( pDbFd->pShm==0 ); + /* If this is a database file (not a journal, master-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif - /* Check to see if a unixShmNode object already exists. Reuse an existing - ** one if present. Create a new one if necessary. +#ifndef NDEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) then record the fact that the database + ** has changed. If the transaction counter is modified, record that + ** fact too. */ - unixEnterMutex(); - pInode = pDbFd->pInode; - pShmNode = pInode->pShmNode; - if( pShmNode==0 ){ - struct stat sStat; /* fstat() info for database file */ - - /* Call fstat() to figure out the permissions on the database file. If - ** a new *-shm file is created, an attempt will be made to create it - ** with the same permissions. The actual permissions the file is created - ** with are subject to the current umask setting. - */ - if( fstat(pDbFd->h, &sStat) ){ - rc = SQLITE_IOERR_FSTAT; - goto shm_open_err; - } - -#ifdef SQLITE_SHM_DIRECTORY - nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 30; -#else - nShmFilename = 5 + (int)strlen(pDbFd->zPath); -#endif - pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); - if( pShmNode==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; + if( pFile->inNormalWrite ){ + pFile->dbUpdate = 1; /* The database has been modified */ + if( offset<=24 && offset+amt>=27 ){ + int rc; + char oldCntr[4]; + SimulateIOErrorBenign(1); + rc = seekAndRead(pFile, 24, oldCntr, 4); + SimulateIOErrorBenign(0); + if( rc!=4 || memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){ + pFile->transCntrChng = 1; /* The transaction counter has changed */ + } } - memset(pShmNode, 0, sizeof(*pShmNode)); - zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; -#ifdef SQLITE_SHM_DIRECTORY - sqlite3_snprintf(nShmFilename, zShmFilename, - SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", - (u32)sStat.st_ino, (u32)sStat.st_dev); -#else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + } #endif - pShmNode->h = -1; - pDbFd->pInode->pShmNode = pShmNode; - pShmNode->pInode = pDbFd->pInode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; - } - pShmNode->h = open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); - if( pShmNode->h<0 ){ - rc = SQLITE_CANTOPEN_BKPT; - goto shm_open_err; - } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ + amt -= wrote; + offset += wrote; + pBuf = &((char*)pBuf)[wrote]; + } + SimulateIOError(( wrote=(-1), amt=1 )); + SimulateDiskfullError(( wrote=0, amt=1 )); - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - rc = SQLITE_OK; - if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( ftruncate(pShmNode->h, 0) ){ - rc = SQLITE_IOERR_SHMOPEN; - } - } - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + if( amt>0 ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ + /* lastErrno set by seekAndWrite */ + return SQLITE_IOERR_WRITE; + }else{ + pFile->lastErrno = 0; /* not a system error */ + return SQLITE_FULL; } - if( rc ) goto shm_open_err; } - /* Make the new connection a child of the unixShmNode */ - p->pShmNode = pShmNode; -#ifdef SQLITE_DEBUG - p->id = pShmNode->nextShmId++; -#endif - pShmNode->nRef++; - pDbFd->pShm = p; - unixLeaveMutex(); - - /* The reference count on pShmNode has already been incremented under - ** the cover of the unixEnterMutex() mutex and the pointer from the - ** new (struct unixShm) object to the pShmNode has been set. All that is - ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. - */ - sqlite3_mutex_enter(pShmNode->mutex); - p->pNext = pShmNode->pFirst; - pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); return SQLITE_OK; - - /* Jump here on any error */ -shm_open_err: - unixShmPurge(pDbFd); /* This call frees pShmNode if required */ - sqlite3_free(p); - unixLeaveMutex(); - return rc; } +#ifdef SQLITE_TEST /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion -** bytes in size. -** -** If an error occurs, an error code is returned and *pp is set to NULL. -** -** Otherwise, if the bExtend parameter is 0 and the requested shared-memory -** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** bExtend is non-zero and the requested shared-memory region has not yet -** been allocated, it is allocated by this function. -** -** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped -** memory and SQLITE_OK returned. +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occurring at the right times. */ -static int unixShmMap( - sqlite3_file *fd, /* Handle open on database file */ - int iRegion, /* Region to retrieve */ - int szRegion, /* Size of regions */ - int bExtend, /* True to extend file if necessary */ - void volatile **pp /* OUT: Mapped memory */ -){ - unixFile *pDbFd = (unixFile*)fd; - unixShm *p; - unixShmNode *pShmNode; - int rc = SQLITE_OK; +SQLITE_API int sqlite3_sync_count = 0; +SQLITE_API int sqlite3_fullsync_count = 0; +#endif - /* If the shared-memory file has not yet been opened, open it now. */ - if( pDbFd->pShm==0 ){ - rc = unixOpenSharedMemory(pDbFd); - if( rc!=SQLITE_OK ) return rc; - } +/* +** We do not trust systems to provide a working fdatasync(). Some do. +** Others do no. To be safe, we will stick with the (slower) fsync(). +** If you know that your system does support fdatasync() correctly, +** then simply compile with -Dfdatasync=fdatasync +*/ +#if !defined(fdatasync) && !defined(__linux__) +# define fdatasync fsync +#endif - p = pDbFd->pShm; - pShmNode = p->pShmNode; - sqlite3_mutex_enter(pShmNode->mutex); - assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); +/* +** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not +** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently +** only available on Mac OS X. But that could change. +*/ +#ifdef F_FULLFSYNC +# define HAVE_FULLFSYNC 1 +#else +# define HAVE_FULLFSYNC 0 +#endif - if( pShmNode->nRegion<=iRegion ){ - char **apNew; /* New apRegion[] array */ - int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ - struct stat sStat; /* Used by fstat() */ - pShmNode->szRegion = szRegion; +/* +** The fsync() system call does not work as advertised on many +** unix systems. The following procedure is an attempt to make +** it work better. +** +** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful +** for testing when we want to run through the test suite quickly. +** You are strongly advised *not* to deploy with SQLITE_NO_SYNC +** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash +** or power failure will likely corrupt the database file. +** +** SQLite sets the dataOnly flag if the size of the file is unchanged. +** The idea behind dataOnly is that it should only write the file content +** to disk, not the inode. We only set dataOnly if the file size is +** unchanged since the file size is part of the inode. However, +** Ted Ts'o tells us that fdatasync() will also write the inode if the +** file size has changed. The only real difference between fdatasync() +** and fsync(), Ted tells us, is that fdatasync() will not flush the +** inode if the mtime or owner or other inode attributes have changed. +** We only care about the file size, not the other file attributes, so +** as far as SQLite is concerned, an fdatasync() is always adequate. +** So, we always use fdatasync() if it is available, regardless of +** the value of the dataOnly flag. +*/ +static int full_fsync(int fd, int fullSync, int dataOnly){ + int rc; - /* The requested region is not mapped into this processes address space. - ** Check to see if it has been allocated (i.e. if the wal-index file is - ** large enough to contain the requested region). - */ - if( fstat(pShmNode->h, &sStat) ){ - rc = SQLITE_IOERR_SHMSIZE; - goto shmpage_out; - } + /* The following "ifdef/elif/else/" block has the same structure as + ** the one below. It is replicated here solely to avoid cluttering + ** up the real code with the UNUSED_PARAMETER() macros. + */ +#ifdef SQLITE_NO_SYNC + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#elif HAVE_FULLFSYNC + UNUSED_PARAMETER(dataOnly); +#else + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#endif - if( sStat.st_sizeh, nByte) ){ - rc = SQLITE_IOERR_SHMSIZE; - goto shmpage_out; - } - } + /* Record the number of times that we do a normal fsync() and + ** FULLSYNC. This is used during testing to verify that this procedure + ** gets called with the correct arguments. + */ +#ifdef SQLITE_TEST + if( fullSync ) sqlite3_fullsync_count++; + sqlite3_sync_count++; +#endif - /* Map the requested memory region into this processes address space. */ - apNew = (char **)sqlite3_realloc( - pShmNode->apRegion, (iRegion+1)*sizeof(char *) - ); - if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; - goto shmpage_out; - } - pShmNode->apRegion = apNew; - while(pShmNode->nRegion<=iRegion){ - void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion - ); - if( pMem==MAP_FAILED ){ - rc = SQLITE_IOERR; - goto shmpage_out; - } - pShmNode->apRegion[pShmNode->nRegion] = pMem; - pShmNode->nRegion++; - } + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + rc = SQLITE_OK; +#elif HAVE_FULLFSYNC + if( fullSync ){ + rc = osFcntl(fd, F_FULLFSYNC, 0); + }else{ + rc = 1; + } + /* If the FULLFSYNC failed, fall back to attempting an fsync(). + ** It shouldn't be possible for fullfsync to fail on the local + ** file system (on OSX), so failure indicates that FULLFSYNC + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid + ** the fcntl call every time sync is called. + */ + if( rc ) rc = fsync(fd); + +#elif defined(__APPLE__) + /* fdatasync() on HFS+ doesn't yet flush the file size if it changed correctly + ** so currently we default to the macro that redefines fdatasync to fsync + */ + rc = fsync(fd); +#else + rc = fdatasync(fd); +#if OS_VXWORKS + if( rc==-1 && errno==ENOTSUP ){ + rc = fsync(fd); } +#endif /* OS_VXWORKS */ +#endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */ -shmpage_out: - if( pShmNode->nRegion>iRegion ){ - *pp = pShmNode->apRegion[iRegion]; - }else{ - *pp = 0; + if( OS_VXWORKS && rc!= -1 ){ + rc = 0; } - sqlite3_mutex_leave(pShmNode->mutex); return rc; } /* -** Change the lock state for a shared-memory segment. +** Make sure all writes to a particular file are committed to disk. ** -** Note that the relationship between SHAREd and EXCLUSIVE locks is a little -** different here than in posix. In xShmLock(), one can go from unlocked -** to shared and back or from unlocked to exclusive and back. But one may -** not go from shared to exclusive or from exclusive to shared. +** If dataOnly==0 then both the file itself and its metadata (file +** size, access time, etc) are synced. If dataOnly!=0 then only the +** file data is synced. +** +** Under Unix, also make sure that the directory entry for the file +** has been created by fsync-ing the directory that contains the file. +** If we do not do this and we encounter a power failure, the directory +** entry for the journal might not exist after we reboot. The next +** SQLite to access the file will not know that the journal exists (because +** the directory entry for the journal was never created) and the transaction +** will not roll back - possibly leading to database corruption. */ -static int unixShmLock( - sqlite3_file *fd, /* Database file holding the shared memory */ - int ofst, /* First lock to acquire or release */ - int n, /* Number of locks to acquire or release */ - int flags /* What to do with the lock */ -){ - unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ - unixShm *p = pDbFd->pShm; /* The shared memory being locked */ - unixShm *pX; /* For looping over all siblings */ - unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ - int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ - - assert( pShmNode==pDbFd->pInode->pShmNode ); - assert( pShmNode->pInode==pDbFd->pInode ); - assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); - assert( n>=1 ); - assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) - || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) - || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) - || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); - assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - - mask = (1<<(ofst+n)) - (1<1 || mask==(1<mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - u16 allMask = 0; /* Mask of locks held by siblings */ +static int unixSync(sqlite3_file *id, int flags){ + int rc; + unixFile *pFile = (unixFile*)id; - /* See if any siblings hold this same lock */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( pX==p ) continue; - assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); - allMask |= pX->sharedMask; - } + int isDataOnly = (flags&SQLITE_SYNC_DATAONLY); + int isFullsync = (flags&0x0F)==SQLITE_SYNC_FULL; - /* Unlock the system-level locks */ - if( (mask & allMask)==0 ){ - rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - }else if( flags & SQLITE_SHM_SHARED ){ - u16 allShared = 0; /* Union of locks held by connections other than "p" */ + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); - /* Find out which shared locks are already held by sibling connections. - ** If any sibling already holds an exclusive lock, go ahead and return - ** SQLITE_BUSY. + assert( pFile ); + OSTRACE(("SYNC %-3d\n", pFile->h)); + rc = full_fsync(pFile->h, isFullsync, isDataOnly); + SimulateIOError( rc=1 ); + if( rc ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); + } + if( pFile->dirfd>=0 ){ + OSTRACE(("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd, + HAVE_FULLFSYNC, isFullsync)); +#ifndef SQLITE_DISABLE_DIRSYNC + /* The directory sync is only attempted if full_fsync is + ** turned off or unavailable. If a full_fsync occurred above, + ** then the directory sync is superfluous. */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - allShared |= pX->sharedMask; + if( (!HAVE_FULLFSYNC || !isFullsync) && full_fsync(pFile->dirfd,0,0) ){ + /* + ** We have received multiple reports of fsync() returning + ** errors when applied to directories on certain file systems. + ** A failed directory sync is not a big deal. So it seems + ** better to ignore the error. Ticket #1657 + */ + /* pFile->lastErrno = errno; */ + /* return SQLITE_IOERR; */ } +#endif + /* Only need to sync once, so close the directory when we are done */ + robust_close(pFile, pFile->dirfd, __LINE__); + pFile->dirfd = -1; + } + return rc; +} - /* Get shared locks at the system level, if necessary */ - if( rc==SQLITE_OK ){ - if( (allShared & mask)==0 ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } - } +/* +** Truncate an open file to a specified size +*/ +static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; + int rc; + assert( pFile ); + SimulateIOError( return SQLITE_IOERR_TRUNCATE ); - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - } + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = robust_ftruncate(pFile->h, (off_t)nByte); + if( rc ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - } - - /* Get the exclusive locks at the system level. Then if successful - ** also mark the local connection as being locked. +#ifndef NDEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) and we truncate the file to zero length, + ** that effectively updates the change counter. This might happen + ** when restoring a database using the backup API from a zero-length + ** source. */ - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n); - if( rc==SQLITE_OK ){ - assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; - } + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; } +#endif + + return SQLITE_OK; } - sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", - p->id, getpid(), p->sharedMask, p->exclMask)); - return rc; } /* -** Implement a memory barrier or memory fence on shared memory. -** -** All loads and stores begun before the barrier must complete before -** any load or store begun after the barrier. +** Determine the current size of a file in bytes */ -static void unixShmBarrier( - sqlite3_file *fd /* Database file holding the shared memory */ -){ - UNUSED_PARAMETER(fd); - unixEnterMutex(); - unixLeaveMutex(); +static int unixFileSize(sqlite3_file *id, i64 *pSize){ + int rc; + struct stat buf; + assert( id ); + rc = osFstat(((unixFile*)id)->h, &buf); + SimulateIOError( rc=1 ); + if( rc!=0 ){ + ((unixFile*)id)->lastErrno = errno; + return SQLITE_IOERR_FSTAT; + } + *pSize = buf.st_size; + + /* When opening a zero-size database, the findInodeInfo() procedure + ** writes a single byte into that file in order to work around a bug + ** in the OS-X msdos filesystem. In order to avoid problems with upper + ** layers, we need to report this file size as zero even though it is + ** really 1. Ticket #3260. + */ + if( *pSize==1 ) *pSize = 0; + + + return SQLITE_OK; } +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) /* -** Close a connection to shared-memory. Delete the underlying -** storage if deleteFlag is true. -** -** If there is no shared memory associated with the connection then this -** routine is a harmless no-op. +** Handler for proxy-locking file-control verbs. Defined below in the +** proxying locking division. */ -static int unixShmUnmap( - sqlite3_file *fd, /* The underlying database file */ - int deleteFlag /* Delete shared-memory if true */ -){ - unixShm *p; /* The connection to be closed */ - unixShmNode *pShmNode; /* The underlying shared-memory file */ - unixShm **pp; /* For looping over sibling connections */ - unixFile *pDbFd; /* The underlying database file */ - - pDbFd = (unixFile*)fd; - p = pDbFd->pShm; - if( p==0 ) return SQLITE_OK; - pShmNode = p->pShmNode; +static int proxyFileControl(sqlite3_file*,int,void*); +#endif - assert( pShmNode==pDbFd->pInode->pShmNode ); - assert( pShmNode->pInode==pDbFd->pInode ); +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. +** +** If the user has configured a chunk-size for this file, it could be +** that the file needs to be extended at this point. Otherwise, the +** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; - /* Remove connection p from the set of connections associated - ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); - for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} - *pp = p->pNext; + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + /* The code below is handling the return value of osFallocate() + ** correctly. posix_fallocate() is defined to "returns zero on success, + ** or an error number on failure". See the manpage for details. */ + int err; + do{ + err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); + }while( err==EINTR ); + if( err ) return SQLITE_IOERR_WRITE; +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. This + ** is the same technique used by glibc to implement posix_fallocate() + ** on systems that do not have a real fallocate() system call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ - /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too */ - unixEnterMutex(); - assert( pShmNode->nRef>0 ); - pShmNode->nRef--; - if( pShmNode->nRef==0 ){ - if( deleteFlag ) unlink(pShmNode->zFilename); - unixShmPurge(pDbFd); + if( robust_ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + while( iWriteeFileLock; + return SQLITE_OK; + } + case SQLITE_LAST_ERRNO: { + *(int*)pArg = ((unixFile*)id)->lastErrno; + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((unixFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); + } +#ifndef NDEBUG + /* The pager calls this method to signal that it has done + ** a rollback and that the database is therefore unchanged and + ** it hence it is OK for the transaction change counter to be + ** unchanged. + */ + case SQLITE_FCNTL_DB_UNCHANGED: { + ((unixFile*)id)->dbUpdate = 0; + return SQLITE_OK; + } +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + case SQLITE_SET_LOCKPROXYFILE: + case SQLITE_GET_LOCKPROXYFILE: { + return proxyFileControl(id,op,pArg); + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + case SQLITE_FCNTL_SYNC_OMITTED: { + return SQLITE_OK; /* A no-op */ + } + } + return SQLITE_NOTFOUND; +} /* -** Here ends the implementation of all sqlite3_file methods. +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. ** -********************** End sqlite3_file Methods ******************************* -******************************************************************************/ +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int unixSectorSize(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return SQLITE_DEFAULT_SECTOR_SIZE; +} /* -** This division contains definitions of sqlite3_io_methods objects that -** implement various file locking strategies. It also contains definitions -** of "finder" functions. A finder-function is used to locate the appropriate -** sqlite3_io_methods object for a particular database file. The pAppData -** field of the sqlite3_vfs VFS objects are initialized to be pointers to -** the correct finder-function for that VFS. -** -** Most finder functions return a pointer to a fixed sqlite3_io_methods -** object. The only interesting finder-function is autolockIoFinder, which -** looks at the filesystem type and tries to guess the best locking -** strategy from that. +** Return the device characteristics for the file. This is always 0 for unix. +*/ +static int unixDeviceCharacteristics(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return 0; +} + +#ifndef SQLITE_OMIT_WAL + + +/* +** Object used to represent an shared memory buffer. ** -** For finder-funtion F, two objects are created: +** When multiple threads all reference the same wal-index, each thread +** has its own unixShm object, but they all point to a single instance +** of this unixShmNode object. In other words, each wal-index is opened +** only once per process. ** -** (1) The real finder-function named "FImpt()". +** Each unixShmNode object is connected to a single unixInodeInfo object. +** We could coalesce this object into unixInodeInfo, but that would mean +** every open file that does not use shared memory (in other words, most +** open files) would have to carry around this extra information. So +** the unixInodeInfo object contains a pointer to this unixShmNode object +** and the unixShmNode object is created only when needed. ** -** (2) A constant pointer to this function named just "F". +** unixMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: ** +** nRef ** -** A pointer to the F pointer is used as the pAppData value for VFS -** objects. We have to do this instead of letting pAppData point -** directly at the finder-function since C90 rules prevent a void* -** from be cast into a function pointer. +** The following fields are read-only after the object is created: +** +** fid +** zFilename ** +** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and +** unixMutexHeld() is true when reading or writing any other field +** in this structure. +*/ +struct unixShmNode { + unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ + sqlite3_mutex *mutex; /* Mutex to access this object */ + char *zFilename; /* Name of the mmapped file */ + int h; /* Open file descriptor */ + int szRegion; /* Size of shared-memory regions */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + char **apRegion; /* Array of mapped shared-memory regions */ + int nRef; /* Number of unixShm objects pointing to this */ + unixShm *pFirst; /* All unixShm objects pointing to this */ +#ifdef SQLITE_DEBUG + u8 exclMask; /* Mask of exclusive locks held */ + u8 sharedMask; /* Mask of shared locks held */ + u8 nextShmId; /* Next available unixShm.id value */ +#endif +}; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. ** -** Each instance of this macro generates two objects: +** The following fields are initialized when this object is created and +** are read-only thereafter: ** -** * A constant sqlite3_io_methods object call METHOD that has locking -** methods CLOSE, LOCK, UNLOCK, CKRESLOCK. +** unixShm.pFile +** unixShm.id ** -** * An I/O method finder function called FINDER that returns a pointer -** to the METHOD object in the previous bullet. +** All other fields are read/write. The unixShm.pFile->mutex must be held +** while accessing any read/write fields. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ -static const sqlite3_io_methods METHOD = { \ - VERSION, /* iVersion */ \ - CLOSE, /* xClose */ \ - unixRead, /* xRead */ \ - unixWrite, /* xWrite */ \ - unixTruncate, /* xTruncate */ \ - unixSync, /* xSync */ \ - unixFileSize, /* xFileSize */ \ - LOCK, /* xLock */ \ - UNLOCK, /* xUnlock */ \ - CKLOCK, /* xCheckReservedLock */ \ - unixFileControl, /* xFileControl */ \ - unixSectorSize, /* xSectorSize */ \ - unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmMap, /* xShmMap */ \ - unixShmLock, /* xShmLock */ \ - unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ -}; \ -static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ - UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ - return &METHOD; \ -} \ -static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ - = FINDER##Impl; +struct unixShm { + unixShmNode *pShmNode; /* The underlying unixShmNode object */ + unixShm *pNext; /* Next unixShm with the same unixShmNode */ + u8 hasMutex; /* True if holding the unixShmNode mutex */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_DEBUG + u8 id; /* Id of this connection within its unixShmNode */ +#endif +}; /* -** Here are all of the sqlite3_io_methods objects for each of the -** locking strategies. Functions that return pointers to these methods -** are also created. +** Constants used for locking */ -IOMETHODS( - posixIoFinder, /* Finder function name */ - posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ - unixClose, /* xClose method */ - unixLock, /* xLock method */ - unixUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ -) -IOMETHODS( - nolockIoFinder, /* Finder function name */ - nolockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - nolockClose, /* xClose method */ - nolockLock, /* xLock method */ - nolockUnlock, /* xUnlock method */ - nolockCheckReservedLock /* xCheckReservedLock method */ -) -IOMETHODS( - dotlockIoFinder, /* Finder function name */ - dotlockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - dotlockClose, /* xClose method */ - dotlockLock, /* xLock method */ - dotlockUnlock, /* xUnlock method */ - dotlockCheckReservedLock /* xCheckReservedLock method */ -) - -#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS -IOMETHODS( - flockIoFinder, /* Finder function name */ - flockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - flockClose, /* xClose method */ - flockLock, /* xLock method */ - flockUnlock, /* xUnlock method */ - flockCheckReservedLock /* xCheckReservedLock method */ -) -#endif - -#if OS_VXWORKS -IOMETHODS( - semIoFinder, /* Finder function name */ - semIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - semClose, /* xClose method */ - semLock, /* xLock method */ - semUnlock, /* xUnlock method */ - semCheckReservedLock /* xCheckReservedLock method */ -) -#endif - -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -IOMETHODS( - afpIoFinder, /* Finder function name */ - afpIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - afpClose, /* xClose method */ - afpLock, /* xLock method */ - afpUnlock, /* xUnlock method */ - afpCheckReservedLock /* xCheckReservedLock method */ -) -#endif +#define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ /* -** The proxy locking method is a "super-method" in the sense that it -** opens secondary file descriptors for the conch and lock files and -** it uses proxy, dot-file, AFP, and flock() locking methods on those -** secondary files. For this reason, the division that implements -** proxy locking is located much further down in the file. But we need -** to go ahead and define the sqlite3_io_methods and finder function -** for proxy locking here. So we forward declare the I/O methods. -*/ -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -static int proxyClose(sqlite3_file*); -static int proxyLock(sqlite3_file*, int); -static int proxyUnlock(sqlite3_file*, int); -static int proxyCheckReservedLock(sqlite3_file*, int*); -IOMETHODS( - proxyIoFinder, /* Finder function name */ - proxyIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - proxyClose, /* xClose method */ - proxyLock, /* xLock method */ - proxyUnlock, /* xUnlock method */ - proxyCheckReservedLock /* xCheckReservedLock method */ -) -#endif - -/* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -IOMETHODS( - nfsIoFinder, /* Finder function name */ - nfsIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ - unixClose, /* xClose method */ - unixLock, /* xLock method */ - nfsUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ -) -#endif - -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -/* -** This "finder" function attempts to determine the best locking strategy -** for the database file "filePath". It then returns the sqlite3_io_methods -** object that implements that strategy. +** Apply posix advisory locks for all bytes from ofst through ofst+n-1. ** -** This is for MacOSX only. +** Locks block if the mask is exactly UNIX_SHM_C and are non-blocking +** otherwise. */ -static const sqlite3_io_methods *autolockIoFinderImpl( - const char *filePath, /* name of the database file */ - unixFile *pNew /* open file object for the database file */ +static int unixShmSystemLock( + unixShmNode *pShmNode, /* Apply locks to this open shared-memory segment */ + int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ + int ofst, /* First byte of the locking range */ + int n /* Number of bytes to lock */ ){ - static const struct Mapping { - const char *zFilesystem; /* Filesystem type name */ - const sqlite3_io_methods *pMethods; /* Appropriate locking method */ - } aMap[] = { - { "hfs", &posixIoMethods }, - { "ufs", &posixIoMethods }, - { "afpfs", &afpIoMethods }, - { "smbfs", &afpIoMethods }, - { "webdav", &nolockIoMethods }, - { 0, 0 } - }; - int i; - struct statfs fsInfo; - struct flock lockInfo; + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ - if( !filePath ){ - /* If filePath==NULL that means we are dealing with a transient file - ** that does not need to be locked. */ - return &nolockIoMethods; - } - if( statfs(filePath, &fsInfo) != -1 ){ - if( fsInfo.f_flags & MNT_RDONLY ){ - return &nolockIoMethods; - } - for(i=0; aMap[i].zFilesystem; i++){ - if( strcmp(fsInfo.f_fstypename, aMap[i].zFilesystem)==0 ){ - return aMap[i].pMethods; - } - } - } + /* Access to the unixShmNode object is serialized by the caller */ + assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); - /* Default case. Handles, amongst others, "nfs". - ** Test byte-range lock using fcntl(). If the call succeeds, - ** assume that the file-system supports POSIX style locks. - */ - lockInfo.l_len = 1; - lockInfo.l_start = 0; - lockInfo.l_whence = SEEK_SET; - lockInfo.l_type = F_RDLCK; - if( fcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { - if( strcmp(fsInfo.f_fstypename, "nfs")==0 ){ - return &nfsIoMethods; - } else { - return &posixIoMethods; - } - }else{ - return &dotlockIoMethods; - } -} -static const sqlite3_io_methods - *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + /* Shared locks never span more than one byte */ + assert( n==1 || lockType!=F_RDLCK ); -#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + /* Locks are within range */ + assert( n>=1 && nh>=0 ){ + /* Initialize the locking parameters */ + memset(&f, 0, sizeof(f)); + f.l_type = lockType; + f.l_whence = SEEK_SET; + f.l_start = ofst; + f.l_len = n; - if( !filePath ){ - /* If filePath==NULL that means we are dealing with a transient file - ** that does not need to be locked. */ - return &nolockIoMethods; + rc = osFcntl(pShmNode->h, F_SETLK, &f); + rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; } - /* Test if fcntl() is supported and use POSIX style locks. - ** Otherwise fall back to the named semaphore method. - */ - lockInfo.l_len = 1; - lockInfo.l_start = 0; - lockInfo.l_whence = SEEK_SET; - lockInfo.l_type = F_RDLCK; - if( fcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { - return &posixIoMethods; + /* Update the global lock state and do debug tracing */ +#ifdef SQLITE_DEBUG + { u16 mask; + OSTRACE(("SHM-LOCK ")); + mask = (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; + } }else{ - return &semIoMethods; + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } + } + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); } +#endif + + return rc; } -static const sqlite3_io_methods - *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; -#endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ /* -** An abstract type for a pointer to a IO method finder function: -*/ -typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); - - -/**************************************************************************** -**************************** sqlite3_vfs methods **************************** +** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0. ** -** This division contains the implementation of methods on the -** sqlite3_vfs object. +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. */ +static void unixShmPurge(unixFile *pFd){ + unixShmNode *p = pFd->pInode->pShmNode; + assert( unixMutexHeld() ); + if( p && p->nRef==0 ){ + int i; + assert( p->pInode==pFd->pInode ); + if( p->mutex ) sqlite3_mutex_free(p->mutex); + for(i=0; inRegion; i++){ + if( p->h>=0 ){ + munmap(p->apRegion[i], p->szRegion); + }else{ + sqlite3_free(p->apRegion[i]); + } + } + sqlite3_free(p->apRegion); + if( p->h>=0 ){ + robust_close(pFd, p->h, __LINE__); + p->h = -1; + } + p->pInode->pShmNode = 0; + sqlite3_free(p); + } +} /* -** Initialize the contents of the unixFile structure pointed to by pId. +** Open a shared-memory area associated with open database file pDbFd. +** This particular implementation uses mmapped files. +** +** The file used to implement shared-memory is in the same directory +** as the open database file and has the same name as the open database +** file with the "-shm" suffix added. For example, if the database file +** is "/home/user1/config.db" then the file that is created and mmapped +** for shared memory will be called "/home/user1/config.db-shm". +** +** Another approach to is to use files in /dev/shm or /dev/tmp or an +** some other tmpfs mount. But if a file in a different directory +** from the database file is used, then differing access permissions +** or a chroot() might cause two different processes on the same +** database to end up using different files for shared memory - +** meaning that their memory would not really be shared - resulting +** in database corruption. Nevertheless, this tmpfs file usage +** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" +** or the equivalent. The use of the SQLITE_SHM_DIRECTORY compile-time +** option results in an incompatible build of SQLite; builds of SQLite +** that with differing SQLITE_SHM_DIRECTORY settings attempt to use the +** same database file at the same time, database corruption will likely +** result. The SQLITE_SHM_DIRECTORY compile-time option is considered +** "unsupported" and may go away in a future SQLite release. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +** +** If the original database file (pDbFd) is using the "unix-excl" VFS +** that means that an exclusive lock is held on the database file and +** that no other processes are able to read or write the database. In +** that case, we do not really need shared memory. No shared memory +** file is created. The shared memory will be simulated with heap memory. */ -static int fillInUnixFile( - sqlite3_vfs *pVfs, /* Pointer to vfs object */ - int h, /* Open file descriptor of file being opened */ - int dirfd, /* Directory file descriptor */ - sqlite3_file *pId, /* Write to the unixFile structure here */ - const char *zFilename, /* Name of the file being opened */ - int noLock, /* Omit locking if true */ - int isDelete /* Delete on close if true */ -){ - const sqlite3_io_methods *pLockingStyle; - unixFile *pNew = (unixFile *)pId; - int rc = SQLITE_OK; - - assert( pNew->pInode==NULL ); +static int unixOpenSharedMemory(unixFile *pDbFd){ + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmNode *pShmNode; /* The underlying mmapped file */ + int rc; /* Result code */ + unixInodeInfo *pInode; /* The inode of fd */ + char *zShmFilename; /* Name of the file used for SHM */ + int nShmFilename; /* Size of the SHM filename in bytes */ - /* Parameter isDelete is only used on vxworks. Express this explicitly - ** here to prevent compiler warnings about unused parameters. - */ - UNUSED_PARAMETER(isDelete); + /* Allocate space for the new unixShm object. */ + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + assert( pDbFd->pShm==0 ); - /* Usually the path zFilename should not be a relative pathname. The - ** exception is when opening the proxy "conch" file in builds that - ** include the special Apple locking styles. + /* Check to see if a unixShmNode object already exists. Reuse an existing + ** one if present. Create a new one if necessary. */ -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE - assert( zFilename==0 || zFilename[0]=='/' - || pVfs->pAppData==(void*)&autolockIoFinder ); -#else - assert( zFilename==0 || zFilename[0]=='/' ); -#endif - - OSTRACE(("OPEN %-3d %s\n", h, zFilename)); - pNew->h = h; - pNew->dirfd = dirfd; - pNew->fileFlags = 0; - pNew->zPath = zFilename; - -#if OS_VXWORKS - pNew->pId = vxworksFindFileId(zFilename); - if( pNew->pId==0 ){ - noLock = 1; - rc = SQLITE_NOMEM; - } -#endif - - if( noLock ){ - pLockingStyle = &nolockIoMethods; - }else{ - pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, pNew); -#if SQLITE_ENABLE_LOCKING_STYLE - /* Cache zFilename in the locking context (AFP and dotlock override) for - ** proxyLock activation is possible (remote proxy is based on db name) - ** zFilename remains valid until file is closed, to support */ - pNew->lockingContext = (void*)zFilename; -#endif - } + unixEnterMutex(); + pInode = pDbFd->pInode; + pShmNode = pInode->pShmNode; + if( pShmNode==0 ){ + struct stat sStat; /* fstat() info for database file */ - if( pLockingStyle == &posixIoMethods -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE - || pLockingStyle == &nfsIoMethods -#endif - ){ - unixEnterMutex(); - rc = findInodeInfo(pNew, &pNew->pInode); - if( rc!=SQLITE_OK ){ - /* If an error occured in findInodeInfo(), close the file descriptor - ** immediately, before releasing the mutex. findInodeInfo() may fail - ** in two scenarios: - ** - ** (a) A call to fstat() failed. - ** (b) A malloc failed. - ** - ** Scenario (b) may only occur if the process is holding no other - ** file descriptors open on the same file. If there were other file - ** descriptors on this file, then no malloc would be required by - ** findInodeInfo(). If this is the case, it is quite safe to close - ** handle h - as it is guaranteed that no posix locks will be released - ** by doing so. - ** - ** If scenario (a) caused the error then things are not so safe. The - ** implicit assumption here is that if fstat() fails, things are in - ** such bad shape that dropping a lock or two doesn't matter much. - */ - close(h); - h = -1; + /* Call fstat() to figure out the permissions on the database file. If + ** a new *-shm file is created, an attempt will be made to create it + ** with the same permissions. The actual permissions the file is created + ** with are subject to the current umask setting. + */ + if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ + rc = SQLITE_IOERR_FSTAT; + goto shm_open_err; } - unixLeaveMutex(); - } -#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - else if( pLockingStyle == &afpIoMethods ){ - /* AFP locking uses the file path so it needs to be included in - ** the afpLockingContext. - */ - afpLockingContext *pCtx; - pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); - if( pCtx==0 ){ +#ifdef SQLITE_SHM_DIRECTORY + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 30; +#else + nShmFilename = 5 + (int)strlen(pDbFd->zPath); +#endif + pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); + if( pShmNode==0 ){ rc = SQLITE_NOMEM; - }else{ - /* NB: zFilename exists and remains valid until the file is closed - ** according to requirement F11141. So we do not need to make a - ** copy of the filename. */ - pCtx->dbPath = zFilename; - pCtx->reserved = 0; - srandomdev(); - unixEnterMutex(); - rc = findInodeInfo(pNew, &pNew->pInode); - if( rc!=SQLITE_OK ){ - sqlite3_free(pNew->lockingContext); - close(h); - h = -1; - } - unixLeaveMutex(); + goto shm_open_err; } - } + memset(pShmNode, 0, sizeof(*pShmNode)); + zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; +#ifdef SQLITE_SHM_DIRECTORY + sqlite3_snprintf(nShmFilename, zShmFilename, + SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", + (u32)sStat.st_ino, (u32)sStat.st_dev); +#else + sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); #endif - - else if( pLockingStyle == &dotlockIoMethods ){ - /* Dotfile locking uses the file path so it needs to be included in - ** the dotlockLockingContext - */ - char *zLockFile; - int nFilename; - nFilename = (int)strlen(zFilename) + 6; - zLockFile = (char *)sqlite3_malloc(nFilename); - if( zLockFile==0 ){ + pShmNode->h = -1; + pDbFd->pInode->pShmNode = pShmNode; + pShmNode->pInode = pDbFd->pInode; + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ rc = SQLITE_NOMEM; - }else{ - sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); + goto shm_open_err; } - pNew->lockingContext = zLockFile; - } -#if OS_VXWORKS - else if( pLockingStyle == &semIoMethods ){ - /* Named semaphore locking uses the file path so it needs to be - ** included in the semLockingContext - */ - unixEnterMutex(); - rc = findInodeInfo(pNew, &pNew->pInode); - if( (rc==SQLITE_OK) && (pNew->pInode->pSem==NULL) ){ - char *zSemName = pNew->pInode->aSemName; - int n; - sqlite3_snprintf(MAX_PATHNAME, zSemName, "/%s.sem", - pNew->pId->zCanonicalName); - for( n=1; zSemName[n]; n++ ) - if( zSemName[n]=='/' ) zSemName[n] = '_'; - pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); - if( pNew->pInode->pSem == SEM_FAILED ){ - rc = SQLITE_NOMEM; - pNew->pInode->aSemName[0] = '\0'; + if( pInode->bProcessLock==0 ){ + pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, + (sStat.st_mode & 0777)); + if( pShmNode->h<0 ){ + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + } + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } } - } - unixLeaveMutex(); - } -#endif - pNew->lastErrno = 0; -#if OS_VXWORKS - if( rc!=SQLITE_OK ){ - if( h>=0 ) close(h); - h = -1; - unlink(zFilename); - isDelete = 0; + /* Check to see if another process is holding the dead-man switch. + ** If not, truncate the file to zero length. + */ + rc = SQLITE_OK; + if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + if( robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); + } + } + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + } + if( rc ) goto shm_open_err; + } } - pNew->isDelete = isDelete; + + /* Make the new connection a child of the unixShmNode */ + p->pShmNode = pShmNode; +#ifdef SQLITE_DEBUG + p->id = pShmNode->nextShmId++; #endif - if( rc!=SQLITE_OK ){ - if( dirfd>=0 ) close(dirfd); /* silent leak if fail, already in error */ - if( h>=0 ) close(h); - }else{ - pNew->pMethod = pLockingStyle; - OpenCounter(+1); - } + pShmNode->nRef++; + pDbFd->pShm = p; + unixLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the unixEnterMutex() mutex and the pointer from the + ** new (struct unixShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); + return SQLITE_OK; + + /* Jump here on any error */ +shm_open_err: + unixShmPurge(pDbFd); /* This call frees pShmNode if required */ + sqlite3_free(p); + unixLeaveMutex(); return rc; } /* -** Open a file descriptor to the directory containing file zFilename. -** If successful, *pFd is set to the opened file descriptor and -** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM -** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined -** value. +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. ** -** If SQLITE_OK is returned, the caller is responsible for closing -** the file descriptor *pFd using close(). +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the bExtend parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. */ -static int openDirectory(const char *zFilename, int *pFd){ - int ii; - int fd = -1; - char zDirname[MAX_PATHNAME+1]; - - sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); - for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); - if( ii>0 ){ - zDirname[ii] = '\0'; - fd = open(zDirname, O_RDONLY|O_BINARY, 0); - if( fd>=0 ){ -#ifdef FD_CLOEXEC - fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); -#endif - OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); - } - } - *pFd = fd; - return (fd>=0?SQLITE_OK:SQLITE_CANTOPEN_BKPT); -} - -/* -** Return the name of a directory in which to put temporary files. -** If no suitable temporary file directory can be found, return NULL. -*/ -static const char *unixTempFileDir(void){ - static const char *azDirs[] = { - 0, - 0, - "/var/tmp", - "/usr/tmp", - "/tmp", - 0 /* List terminator */ - }; - unsigned int i; - struct stat buf; - const char *zDir = 0; +static int unixShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + unixFile *pDbFd = (unixFile*)fd; + unixShm *p; + unixShmNode *pShmNode; + int rc = SQLITE_OK; - azDirs[0] = sqlite3_temp_directory; - if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - for(i=0; ipShm==0 ){ + rc = unixOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; } - return zDir; -} -/* -** Create a temporary file name in zBuf. zBuf must be allocated -** by the calling process and must be big enough to hold at least -** pVfs->mxPathname bytes. -*/ -static int unixGetTempname(int nBuf, char *zBuf){ - static const unsigned char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - unsigned int i, j; - const char *zDir; + p = pDbFd->pShm; + pShmNode = p->pShmNode; + sqlite3_mutex_enter(pShmNode->mutex); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); - /* It's odd to simulate an io-error here, but really this is just - ** using the io-error infrastructure to test that SQLite handles this - ** function failing. - */ - SimulateIOError( return SQLITE_IOERR ); + if( pShmNode->nRegion<=iRegion ){ + char **apNew; /* New apRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + struct stat sStat; /* Used by fstat() */ - zDir = unixTempFileDir(); - if( zDir==0 ) zDir = "."; + pShmNode->szRegion = szRegion; - /* Check that the output buffer is large enough for the temporary file - ** name. If it is not, return SQLITE_ERROR. - */ - if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 17) >= (size_t)nBuf ){ - return SQLITE_ERROR; - } + if( pShmNode->h>=0 ){ + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + if( osFstat(pShmNode->h, &sStat) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + + if( sStat.st_sizeh, nByte) ){ + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", + pShmNode->zFilename); + goto shmpage_out; + } + } + } - do{ - sqlite3_snprintf(nBuf-17, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); - j = (int)strlen(zBuf); - sqlite3_randomness(15, &zBuf[j]); - for(i=0; i<15; i++, j++){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + /* Map the requested memory region into this processes address space. */ + apNew = (char **)sqlite3_realloc( + pShmNode->apRegion, (iRegion+1)*sizeof(char *) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; } - zBuf[j] = 0; - }while( access(zBuf,0)==0 ); - return SQLITE_OK; -} + pShmNode->apRegion = apNew; + while(pShmNode->nRegion<=iRegion){ + void *pMem; + if( pShmNode->h>=0 ){ + pMem = mmap(0, szRegion, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion + ); + if( pMem==MAP_FAILED ){ + rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); + goto shmpage_out; + } + }else{ + pMem = sqlite3_malloc(szRegion); + if( pMem==0 ){ + rc = SQLITE_NOMEM; + goto shmpage_out; + } + memset(pMem, 0, szRegion); + } + pShmNode->apRegion[pShmNode->nRegion] = pMem; + pShmNode->nRegion++; + } + } -#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) -/* -** Routine to transform a unixFile into a proxy-locking unixFile. -** Implementation in the proxy-lock division, but used by unixOpen() -** if SQLITE_PREFER_PROXY_LOCKING is defined. -*/ -static int proxyTransformUnixFile(unixFile*, const char*); -#endif +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + *pp = pShmNode->apRegion[iRegion]; + }else{ + *pp = 0; + } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; +} /* -** Search for an unused file descriptor that was opened on the database -** file (not a journal or master-journal file) identified by pathname -** zPath with SQLITE_OPEN_XXX flags matching those passed as the second -** argument to this function. -** -** Such a file descriptor may exist if a database connection was closed -** but the associated file descriptor could not be closed because some -** other file descriptor open on the same file is holding a file-lock. -** Refer to comments in the unixClose() function and the lengthy comment -** describing "Posix Advisory Locking" at the start of this file for -** further details. Also, ticket #4018. +** Change the lock state for a shared-memory segment. ** -** If a suitable file descriptor is found, then it is returned. If no -** such file descriptor is located, -1 is returned. +** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** different here than in posix. In xShmLock(), one can go from unlocked +** to shared and back or from unlocked to exclusive and back. But one may +** not go from shared to exclusive or from exclusive to shared. */ -static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ - UnixUnusedFd *pUnused = 0; +static int unixShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ + unixShm *p = pDbFd->pShm; /* The shared memory being locked */ + unixShm *pX; /* For looping over all siblings */ + unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ - /* Do not search for an unused file descriptor on vxworks. Not because - ** vxworks would not benefit from the change (it might, we're not sure), - ** but because no way to test it is currently available. It is better - ** not to risk breaking vxworks support for the sake of such an obscure - ** feature. */ -#if !OS_VXWORKS - struct stat sStat; /* Results of stat() call */ + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); - /* A stat() call may fail for various reasons. If this happens, it is - ** almost certain that an open() call on the same path will also fail. - ** For this reason, if an error occurs in the stat() call here, it is - ** ignored and -1 is returned. The caller will try to open a new file - ** descriptor on the same path, fail, and return an error to SQLite. - ** - ** Even if a subsequent open() call does succeed, the consequences of - ** not searching for a resusable file descriptor are not dire. */ - if( 0==stat(zPath, &sStat) ){ - unixInodeInfo *pInode; + mask = (1<<(ofst+n)) - (1<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ - unixEnterMutex(); - pInode = inodeList; - while( pInode && (pInode->fileId.dev!=sStat.st_dev - || pInode->fileId.ino!=sStat.st_ino) ){ - pInode = pInode->pNext; + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; } - if( pInode ){ - UnixUnusedFd **pp; - for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); - pUnused = *pp; - if( pUnused ){ - *pp = pUnused->pNext; - } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n); + }else{ + rc = SQLITE_OK; } - unixLeaveMutex(); - } -#endif /* if !OS_VXWORKS */ - return pUnused; -} -/* -** This function is called by unixOpen() to determine the unix permissions -** to create new files with. If no error occurs, then SQLITE_OK is returned -** and a value suitable for passing as the third argument to open(2) is -** written to *pMode. If an IO error occurs, an SQLite error code is -** returned and the value of *pMode is not modified. -** -** If the file being opened is a temporary file, it is always created with -** the octal permissions 0600 (read/writable by owner only). If the file -** is a database or master journal file, it is created with the permissions -** mask SQLITE_DEFAULT_FILE_PERMISSIONS. -** -** Finally, if the file being opened is a WAL or regular journal file, then -** this function queries the file-system for the permissions on the -** corresponding database file and sets *pMode to this value. Whenever -** possible, WAL and journal files are created using the same permissions -** as the associated database file. -*/ -static int findCreateFileMode( - const char *zPath, /* Path of file (possibly) being created */ - int flags, /* Flags passed as 4th argument to xOpen() */ - mode_t *pMode /* OUT: Permissions to open file with */ -){ - int rc = SQLITE_OK; /* Return Code */ - if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ - char zDb[MAX_PATHNAME+1]; /* Database file path */ - int nDb; /* Number of valid bytes in zDb */ - struct stat sStat; /* Output of stat() on database file */ + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ - /* zPath is a path to a WAL or journal file. The following block derives - ** the path to the associated database file from zPath. This block handles - ** the following naming conventions: - ** - ** "-journal" - ** "-wal" - ** "-journal-NNNN" - ** "-wal-NNNN" - ** - ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are - ** used by the test_multiplex.c module. + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. */ - nDb = sqlite3Strlen30(zPath) - 1; - while( nDb>0 && zPath[nDb]!='l' ) nDb--; - nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7); - memcpy(zDb, zPath, nDb); - zDb[nDb] = '\0'; + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } - if( 0==stat(zDb, &sStat) ){ - *pMode = sStat.st_mode & 0777; - }else{ - rc = SQLITE_IOERR_FSTAT; + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; } - }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ - *pMode = 0600; }else{ - *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } } + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", + p->id, getpid(), p->sharedMask, p->exclMask)); return rc; } /* -** Open the file zPath. -** -** Previously, the SQLite OS layer used three functions in place of this -** one: -** -** sqlite3OsOpenReadWrite(); -** sqlite3OsOpenReadOnly(); -** sqlite3OsOpenExclusive(); -** -** These calls correspond to the following combinations of flags: +** Implement a memory barrier or memory fence on shared memory. ** -** ReadWrite() -> (READWRITE | CREATE) -** ReadOnly() -> (READONLY) -** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void unixShmBarrier( + sqlite3_file *fd /* Database file holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + unixEnterMutex(); + unixLeaveMutex(); +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. ** -** The old OpenExclusive() accepted a boolean argument - "delFlag". If -** true, the file was configured to be automatically deleted when the -** file handle closed. To achieve the same effect using this new -** interface, add the DELETEONCLOSE flag to those specified above for -** OpenExclusive(). +** If there is no shared memory associated with the connection then this +** routine is a harmless no-op. */ -static int unixOpen( - sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */ - const char *zPath, /* Pathname of file to be opened */ - sqlite3_file *pFile, /* The file descriptor to be filled in */ - int flags, /* Input flags to control the opening */ - int *pOutFlags /* Output flags returned to SQLite core */ +static int unixShmUnmap( + sqlite3_file *fd, /* The underlying database file */ + int deleteFlag /* Delete shared-memory if true */ ){ - unixFile *p = (unixFile *)pFile; - int fd = -1; /* File descriptor returned by open() */ - int dirfd = -1; /* Directory file descriptor */ - int openFlags = 0; /* Flags to pass to open() */ - int eType = flags&0xFFFFFF00; /* Type of file to open */ - int noLock; /* True to omit locking primitives */ - int rc = SQLITE_OK; /* Function Return Code */ + unixShm *p; /* The connection to be closed */ + unixShmNode *pShmNode; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + unixFile *pDbFd; /* The underlying database file */ - int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); - int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); - int isCreate = (flags & SQLITE_OPEN_CREATE); - int isReadonly = (flags & SQLITE_OPEN_READONLY); - int isReadWrite = (flags & SQLITE_OPEN_READWRITE); -#if SQLITE_ENABLE_LOCKING_STYLE - int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); -#endif + pDbFd = (unixFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; - /* If creating a master or main-file journal, this function will open - ** a file-descriptor on the directory too. The first time unixSync() - ** is called the directory file descriptor will be fsync()ed and close()d. - */ - int isOpenDirectory = (isCreate && ( - eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_MAIN_JOURNAL - || eType==SQLITE_OPEN_WAL - )); + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); - /* If argument zPath is a NULL pointer, this function is required to open - ** a temporary file. Use this buffer to store the file name in. - */ - char zTmpname[MAX_PATHNAME+1]; - const char *zName = zPath; + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; - /* Check the following statements are true: - ** - ** (a) Exactly one of the READWRITE and READONLY flags must be set, and - ** (b) if CREATE is set, then READWRITE must also be set, and - ** (c) if EXCLUSIVE is set, then CREATE must also be set. - ** (d) if DELETEONCLOSE is set, then CREATE must also be set. - */ - assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); - assert(isCreate==0 || isReadWrite); - assert(isExclusive==0 || isCreate); - assert(isDelete==0 || isCreate); + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->mutex); - /* The main DB, main journal, WAL file and master journal are never - ** automatically deleted. Nor are they ever temporary files. */ - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + unixEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + if( deleteFlag && pShmNode->h>=0 ) unlink(pShmNode->zFilename); + unixShmPurge(pDbFd); + } + unixLeaveMutex(); - /* Assert that the upper layer has set one of the "file-type" flags. */ - assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB - || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL - ); + return SQLITE_OK; +} - memset(p, 0, sizeof(unixFile)); - if( eType==SQLITE_OPEN_MAIN_DB ){ - UnixUnusedFd *pUnused; - pUnused = findReusableFd(zName, flags); - if( pUnused ){ - fd = pUnused->fd; - }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); - if( !pUnused ){ - return SQLITE_NOMEM; - } - } - p->pUnused = pUnused; - }else if( !zName ){ - /* If zName is NULL, the upper layer is requesting a temp file. */ - assert(isDelete && !isOpenDirectory); - rc = unixGetTempname(MAX_PATHNAME+1, zTmpname); - if( rc!=SQLITE_OK ){ - return rc; - } - zName = zTmpname; - } +#else +# define unixShmMap 0 +# define unixShmLock 0 +# define unixShmBarrier 0 +# define unixShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ - /* Determine the value of the flags parameter passed to POSIX function - ** open(). These must be calculated even if open() is not called, as - ** they may be stored as part of the file handle and used by the - ** 'conch file' locking functions later on. */ - if( isReadonly ) openFlags |= O_RDONLY; - if( isReadWrite ) openFlags |= O_RDWR; - if( isCreate ) openFlags |= O_CREAT; - if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); - openFlags |= (O_LARGEFILE|O_BINARY); +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ - if( fd<0 ){ - mode_t openMode; /* Permissions to create file with */ - rc = findCreateFileMode(zName, flags, &openMode); - if( rc!=SQLITE_OK ){ - assert( !p->pUnused ); - assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); - return rc; - } - fd = open(zName, openFlags, openMode); - OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); - if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ - /* Failed to open the file for read/write access. Try read-only. */ - flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); - openFlags &= ~(O_RDWR|O_CREAT); - flags |= SQLITE_OPEN_READONLY; - openFlags |= O_RDONLY; - fd = open(zName, openFlags, openMode); - } - if( fd<0 ){ - rc = SQLITE_CANTOPEN_BKPT; - goto open_finished; - } - } - assert( fd>=0 ); - if( pOutFlags ){ - *pOutFlags = flags; - } +/* +** This division contains definitions of sqlite3_io_methods objects that +** implement various file locking strategies. It also contains definitions +** of "finder" functions. A finder-function is used to locate the appropriate +** sqlite3_io_methods object for a particular database file. The pAppData +** field of the sqlite3_vfs VFS objects are initialized to be pointers to +** the correct finder-function for that VFS. +** +** Most finder functions return a pointer to a fixed sqlite3_io_methods +** object. The only interesting finder-function is autolockIoFinder, which +** looks at the filesystem type and tries to guess the best locking +** strategy from that. +** +** For finder-funtion F, two objects are created: +** +** (1) The real finder-function named "FImpt()". +** +** (2) A constant pointer to this function named just "F". +** +** +** A pointer to the F pointer is used as the pAppData value for VFS +** objects. We have to do this instead of letting pAppData point +** directly at the finder-function since C90 rules prevent a void* +** from be cast into a function pointer. +** +** +** Each instance of this macro generates two objects: +** +** * A constant sqlite3_io_methods object call METHOD that has locking +** methods CLOSE, LOCK, UNLOCK, CKRESLOCK. +** +** * An I/O method finder function called FINDER that returns a pointer +** to the METHOD object in the previous bullet. +*/ +#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +static const sqlite3_io_methods METHOD = { \ + VERSION, /* iVersion */ \ + CLOSE, /* xClose */ \ + unixRead, /* xRead */ \ + unixWrite, /* xWrite */ \ + unixTruncate, /* xTruncate */ \ + unixSync, /* xSync */ \ + unixFileSize, /* xFileSize */ \ + LOCK, /* xLock */ \ + UNLOCK, /* xUnlock */ \ + CKLOCK, /* xCheckReservedLock */ \ + unixFileControl, /* xFileControl */ \ + unixSectorSize, /* xSectorSize */ \ + unixDeviceCharacteristics, /* xDeviceCapabilities */ \ + unixShmMap, /* xShmMap */ \ + unixShmLock, /* xShmLock */ \ + unixShmBarrier, /* xShmBarrier */ \ + unixShmUnmap /* xShmUnmap */ \ +}; \ +static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ + UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ + return &METHOD; \ +} \ +static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ + = FINDER##Impl; - if( p->pUnused ){ - p->pUnused->fd = fd; - p->pUnused->flags = flags; - } +/* +** Here are all of the sqlite3_io_methods objects for each of the +** locking strategies. Functions that return pointers to these methods +** are also created. +*/ +IOMETHODS( + posixIoFinder, /* Finder function name */ + posixIoMethods, /* sqlite3_io_methods object name */ + 2, /* shared memory is enabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + unixUnlock, /* xUnlock method */ + unixCheckReservedLock /* xCheckReservedLock method */ +) +IOMETHODS( + nolockIoFinder, /* Finder function name */ + nolockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + nolockClose, /* xClose method */ + nolockLock, /* xLock method */ + nolockUnlock, /* xUnlock method */ + nolockCheckReservedLock /* xCheckReservedLock method */ +) +IOMETHODS( + dotlockIoFinder, /* Finder function name */ + dotlockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + dotlockClose, /* xClose method */ + dotlockLock, /* xLock method */ + dotlockUnlock, /* xUnlock method */ + dotlockCheckReservedLock /* xCheckReservedLock method */ +) - if( isDelete ){ -#if OS_VXWORKS - zPath = zName; -#else - unlink(zName); +#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS +IOMETHODS( + flockIoFinder, /* Finder function name */ + flockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + flockClose, /* xClose method */ + flockLock, /* xLock method */ + flockUnlock, /* xUnlock method */ + flockCheckReservedLock /* xCheckReservedLock method */ +) #endif - } -#if SQLITE_ENABLE_LOCKING_STYLE - else{ - p->openFlags = openFlags; - } + +#if OS_VXWORKS +IOMETHODS( + semIoFinder, /* Finder function name */ + semIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + semClose, /* xClose method */ + semLock, /* xLock method */ + semUnlock, /* xUnlock method */ + semCheckReservedLock /* xCheckReservedLock method */ +) #endif - if( isOpenDirectory ){ - rc = openDirectory(zPath, &dirfd); - if( rc!=SQLITE_OK ){ - /* It is safe to close fd at this point, because it is guaranteed not - ** to be open on a database file. If it were open on a database file, - ** it would not be safe to close as this would release any locks held - ** on the file by this process. */ - assert( eType!=SQLITE_OPEN_MAIN_DB ); - close(fd); /* silently leak if fail, already in error */ - goto open_finished; - } - } +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + afpIoFinder, /* Finder function name */ + afpIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + afpClose, /* xClose method */ + afpLock, /* xLock method */ + afpUnlock, /* xUnlock method */ + afpCheckReservedLock /* xCheckReservedLock method */ +) +#endif -#ifdef FD_CLOEXEC - fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +/* +** The proxy locking method is a "super-method" in the sense that it +** opens secondary file descriptors for the conch and lock files and +** it uses proxy, dot-file, AFP, and flock() locking methods on those +** secondary files. For this reason, the division that implements +** proxy locking is located much further down in the file. But we need +** to go ahead and define the sqlite3_io_methods and finder function +** for proxy locking here. So we forward declare the I/O methods. +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static int proxyClose(sqlite3_file*); +static int proxyLock(sqlite3_file*, int); +static int proxyUnlock(sqlite3_file*, int); +static int proxyCheckReservedLock(sqlite3_file*, int*); +IOMETHODS( + proxyIoFinder, /* Finder function name */ + proxyIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + proxyClose, /* xClose method */ + proxyLock, /* xLock method */ + proxyUnlock, /* xUnlock method */ + proxyCheckReservedLock /* xCheckReservedLock method */ +) #endif - noLock = eType!=SQLITE_OPEN_MAIN_DB; +/* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + nfsIoFinder, /* Finder function name */ + nfsIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + nfsUnlock, /* xUnlock method */ + unixCheckReservedLock /* xCheckReservedLock method */ +) +#endif - -#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. +** +** This is for MacOSX only. +*/ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* open file object for the database file */ +){ + static const struct Mapping { + const char *zFilesystem; /* Filesystem type name */ + const sqlite3_io_methods *pMethods; /* Appropriate locking method */ + } aMap[] = { + { "hfs", &posixIoMethods }, + { "ufs", &posixIoMethods }, + { "afpfs", &afpIoMethods }, + { "smbfs", &afpIoMethods }, + { "webdav", &nolockIoMethods }, + { 0, 0 } + }; + int i; struct statfs fsInfo; - if( fstatfs(fd, &fsInfo) == -1 ){ - ((unixFile*)pFile)->lastErrno = errno; - if( dirfd>=0 ) close(dirfd); /* silently leak if fail, in error */ - close(fd); /* silently leak if fail, in error */ - return SQLITE_IOERR_ACCESS; - } - if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { - ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; - } -#endif - -#if SQLITE_ENABLE_LOCKING_STYLE -#if SQLITE_PREFER_PROXY_LOCKING - isAutoProxy = 1; -#endif - if( isAutoProxy && (zPath!=NULL) && (!noLock) && pVfs->xOpen ){ - char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); - int useProxy = 0; + struct flock lockInfo; - /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means - ** never use proxy, NULL means use proxy for non-local files only. */ - if( envforce!=NULL ){ - useProxy = atoi(envforce)>0; - }else{ - struct statfs fsInfo; - if( statfs(zPath, &fsInfo) == -1 ){ - /* In theory, the close(fd) call is sub-optimal. If the file opened - ** with fd is a database file, and there are other connections open - ** on that file that are currently holding advisory locks on it, - ** then the call to close() will cancel those locks. In practice, - ** we're assuming that statfs() doesn't fail very often. At least - ** not while other file descriptors opened by the same process on - ** the same file are working. */ - p->lastErrno = errno; - if( dirfd>=0 ){ - close(dirfd); /* silently leak if fail, in error */ - } - close(fd); /* silently leak if fail, in error */ - rc = SQLITE_IOERR_ACCESS; - goto open_finished; - } - useProxy = !(fsInfo.f_flags&MNT_LOCAL); + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + if( statfs(filePath, &fsInfo) != -1 ){ + if( fsInfo.f_flags & MNT_RDONLY ){ + return &nolockIoMethods; } - if( useProxy ){ - rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, isDelete); - if( rc==SQLITE_OK ){ - rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); - if( rc!=SQLITE_OK ){ - /* Use unixClose to clean up the resources added in fillInUnixFile - ** and clear all the structure's references. Specifically, - ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op - */ - unixClose(pFile); - return rc; - } + for(i=0; aMap[i].zFilesystem; i++){ + if( strcmp(fsInfo.f_fstypename, aMap[i].zFilesystem)==0 ){ + return aMap[i].pMethods; } - goto open_finished; } } -#endif - - rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, isDelete); -open_finished: - if( rc!=SQLITE_OK ){ - sqlite3_free(p->pUnused); - } - return rc; -} - -/* -** Delete the file at zPath. If the dirSync argument is true, fsync() -** the directory after deleting the file. -*/ -static int unixDelete( - sqlite3_vfs *NotUsed, /* VFS containing this as the xDelete method */ - const char *zPath, /* Name of file to be deleted */ - int dirSync /* If true, fsync() directory after deleting file */ -){ - int rc = SQLITE_OK; - UNUSED_PARAMETER(NotUsed); - SimulateIOError(return SQLITE_IOERR_DELETE); - if( unlink(zPath)==(-1) && errno!=ENOENT ){ - return SQLITE_IOERR_DELETE; - } -#ifndef SQLITE_DISABLE_DIRSYNC - if( dirSync ){ - int fd; - rc = openDirectory(zPath, &fd); - if( rc==SQLITE_OK ){ -#if OS_VXWORKS - if( fsync(fd)==-1 ) -#else - if( fsync(fd) ) -#endif - { - rc = SQLITE_IOERR_DIR_FSYNC; - } - if( close(fd)&&!rc ){ - rc = SQLITE_IOERR_DIR_CLOSE; - } + /* Default case. Handles, amongst others, "nfs". + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + if( strcmp(fsInfo.f_fstypename, "nfs")==0 ){ + return &nfsIoMethods; + } else { + return &posixIoMethods; } + }else{ + return &dotlockIoMethods; } -#endif - return rc; } +static const sqlite3_io_methods + *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; -/* -** Test the existance of or access permissions of file zPath. The -** test performed depends on the value of flags: -** -** SQLITE_ACCESS_EXISTS: Return 1 if the file exists -** SQLITE_ACCESS_READWRITE: Return 1 if the file is read and writable. -** SQLITE_ACCESS_READONLY: Return 1 if the file is readable. +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + +#if OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE +/* +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. ** -** Otherwise return 0. +** This is for VXWorks only. */ -static int unixAccess( - sqlite3_vfs *NotUsed, /* The VFS containing this xAccess method */ - const char *zPath, /* Path of the file to examine */ - int flags, /* What do we want to learn about the zPath file? */ - int *pResOut /* Write result boolean here */ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* the open file object */ ){ - int amode = 0; - UNUSED_PARAMETER(NotUsed); - SimulateIOError( return SQLITE_IOERR_ACCESS; ); - switch( flags ){ - case SQLITE_ACCESS_EXISTS: - amode = F_OK; - break; - case SQLITE_ACCESS_READWRITE: - amode = W_OK|R_OK; - break; - case SQLITE_ACCESS_READ: - amode = R_OK; - break; + struct flock lockInfo; - default: - assert(!"Invalid flags argument"); + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; } - *pResOut = (access(zPath, amode)==0); - if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ - struct stat buf; - if( 0==stat(zPath, &buf) && buf.st_size==0 ){ - *pResOut = 0; - } + + /* Test if fcntl() is supported and use POSIX style locks. + ** Otherwise fall back to the named semaphore method. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + return &posixIoMethods; + }else{ + return &semIoMethods; } - return SQLITE_OK; } +static const sqlite3_io_methods + *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; +#endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ /* -** Turn a relative pathname into a full pathname. The relative path -** is stored as a nul-terminated string in the buffer pointed to by -** zPath. +** An abstract type for a pointer to a IO method finder function: +*/ +typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); + + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** ** -** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes -** (in this case, MAX_PATHNAME bytes). The full-path is written to -** this buffer before returning. +** This division contains the implementation of methods on the +** sqlite3_vfs object. */ -static int unixFullPathname( - sqlite3_vfs *pVfs, /* Pointer to vfs object */ - const char *zPath, /* Possibly relative input path */ - int nOut, /* Size of output buffer in bytes */ - char *zOut /* Output buffer */ + +/* +** Initialize the contents of the unixFile structure pointed to by pId. +*/ +static int fillInUnixFile( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + int h, /* Open file descriptor of file being opened */ + int dirfd, /* Directory file descriptor */ + sqlite3_file *pId, /* Write to the unixFile structure here */ + const char *zFilename, /* Name of the file being opened */ + int noLock, /* Omit locking if true */ + int isDelete, /* Delete on close if true */ + int isReadOnly /* True if the file is opened read-only */ ){ + const sqlite3_io_methods *pLockingStyle; + unixFile *pNew = (unixFile *)pId; + int rc = SQLITE_OK; - /* It's odd to simulate an io-error here, but really this is just - ** using the io-error infrastructure to test that SQLite handles this - ** function failing. This function could fail if, for example, the - ** current working directory has been unlinked. + assert( pNew->pInode==NULL ); + + /* Parameter isDelete is only used on vxworks. Express this explicitly + ** here to prevent compiler warnings about unused parameters. */ - SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(isDelete); - assert( pVfs->mxPathname==MAX_PATHNAME ); - UNUSED_PARAMETER(pVfs); + /* Usually the path zFilename should not be a relative pathname. The + ** exception is when opening the proxy "conch" file in builds that + ** include the special Apple locking styles. + */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + assert( zFilename==0 || zFilename[0]=='/' + || pVfs->pAppData==(void*)&autolockIoFinder ); +#else + assert( zFilename==0 || zFilename[0]=='/' ); +#endif - zOut[nOut-1] = '\0'; - if( zPath[0]=='/' ){ - sqlite3_snprintf(nOut, zOut, "%s", zPath); + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); + pNew->h = h; + pNew->dirfd = dirfd; + pNew->zPath = zFilename; + if( memcmp(pVfs->zName,"unix-excl",10)==0 ){ + pNew->ctrlFlags = UNIXFILE_EXCL; }else{ - int nCwd; - if( getcwd(zOut, nOut-1)==0 ){ - return SQLITE_CANTOPEN_BKPT; - } - nCwd = (int)strlen(zOut); - sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); + pNew->ctrlFlags = 0; + } + if( isReadOnly ){ + pNew->ctrlFlags |= UNIXFILE_RDONLY; } - return SQLITE_OK; -} - -#ifndef SQLITE_OMIT_LOAD_EXTENSION -/* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ -#include -static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ - UNUSED_PARAMETER(NotUsed); - return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); -} +#if OS_VXWORKS + pNew->pId = vxworksFindFileId(zFilename); + if( pNew->pId==0 ){ + noLock = 1; + rc = SQLITE_NOMEM; + } +#endif -/* -** SQLite calls this function immediately after a call to unixDlSym() or -** unixDlOpen() fails (returns a null pointer). If a more detailed error -** message is available, it is written to zBufOut. If no error message -** is available, zBufOut is left unmodified and SQLite uses a default -** error message. -*/ -static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ - const char *zErr; - UNUSED_PARAMETER(NotUsed); - unixEnterMutex(); - zErr = dlerror(); - if( zErr ){ - sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); + if( noLock ){ + pLockingStyle = &nolockIoMethods; + }else{ + pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, pNew); +#if SQLITE_ENABLE_LOCKING_STYLE + /* Cache zFilename in the locking context (AFP and dotlock override) for + ** proxyLock activation is possible (remote proxy is based on db name) + ** zFilename remains valid until file is closed, to support */ + pNew->lockingContext = (void*)zFilename; +#endif } - unixLeaveMutex(); -} -static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ - /* - ** GCC with -pedantic-errors says that C90 does not allow a void* to be - ** cast into a pointer to a function. And yet the library dlsym() routine - ** returns a void* which is really a pointer to a function. So how do we - ** use dlsym() with -pedantic-errors? - ** - ** Variable x below is defined to be a pointer to a function taking - ** parameters void* and const char* and returning a pointer to a function. - ** We initialize x by assigning it a pointer to the dlsym() function. - ** (That assignment requires a cast.) Then we call the function that - ** x points to. - ** - ** This work-around is unlikely to work correctly on any system where - ** you really cannot cast a function pointer into void*. But then, on the - ** other hand, dlsym() will not work on such a system either, so we have - ** not really lost anything. - */ - void (*(*x)(void*,const char*))(void); - UNUSED_PARAMETER(NotUsed); - x = (void(*(*)(void*,const char*))(void))dlsym; - return (*x)(p, zSym); -} -static void unixDlClose(sqlite3_vfs *NotUsed, void *pHandle){ - UNUSED_PARAMETER(NotUsed); - dlclose(pHandle); -} -#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ - #define unixDlOpen 0 - #define unixDlError 0 - #define unixDlSym 0 - #define unixDlClose 0 -#endif -/* -** Write nBuf bytes of random data to the supplied buffer zBuf. -*/ -static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ - UNUSED_PARAMETER(NotUsed); - assert((size_t)nBuf>=(sizeof(time_t)+sizeof(int))); + if( pLockingStyle == &posixIoMethods +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + || pLockingStyle == &nfsIoMethods +#endif + ){ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + /* If an error occured in findInodeInfo(), close the file descriptor + ** immediately, before releasing the mutex. findInodeInfo() may fail + ** in two scenarios: + ** + ** (a) A call to fstat() failed. + ** (b) A malloc failed. + ** + ** Scenario (b) may only occur if the process is holding no other + ** file descriptors open on the same file. If there were other file + ** descriptors on this file, then no malloc would be required by + ** findInodeInfo(). If this is the case, it is quite safe to close + ** handle h - as it is guaranteed that no posix locks will be released + ** by doing so. + ** + ** If scenario (a) caused the error then things are not so safe. The + ** implicit assumption here is that if fstat() fails, things are in + ** such bad shape that dropping a lock or two doesn't matter much. + */ + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } - /* We have to initialize zBuf to prevent valgrind from reporting - ** errors. The reports issued by valgrind are incorrect - we would - ** prefer that the randomness be increased by making use of the - ** uninitialized space in zBuf - but valgrind errors tend to worry - ** some users. Rather than argue, it seems easier just to initialize - ** the whole array and silence valgrind, even if that means less randomness - ** in the random seed. - ** - ** When testing, initializing zBuf[] to zero is all we do. That means - ** that we always use the same random number sequence. This makes the - ** tests repeatable. - */ - memset(zBuf, 0, nBuf); -#if !defined(SQLITE_TEST) - { - int pid, fd; - fd = open("/dev/urandom", O_RDONLY); - if( fd<0 ){ - time_t t; - time(&t); - memcpy(zBuf, &t, sizeof(t)); - pid = getpid(); - memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid)); - assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); - nBuf = sizeof(t) + sizeof(pid); +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + else if( pLockingStyle == &afpIoMethods ){ + /* AFP locking uses the file path so it needs to be included in + ** the afpLockingContext. + */ + afpLockingContext *pCtx; + pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); + if( pCtx==0 ){ + rc = SQLITE_NOMEM; }else{ - nBuf = read(fd, zBuf, nBuf); - close(fd); + /* NB: zFilename exists and remains valid until the file is closed + ** according to requirement F11141. So we do not need to make a + ** copy of the filename. */ + pCtx->dbPath = zFilename; + pCtx->reserved = 0; + srandomdev(); + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew->lockingContext); + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); } } #endif - return nBuf; -} + else if( pLockingStyle == &dotlockIoMethods ){ + /* Dotfile locking uses the file path so it needs to be included in + ** the dotlockLockingContext + */ + char *zLockFile; + int nFilename; + nFilename = (int)strlen(zFilename) + 6; + zLockFile = (char *)sqlite3_malloc(nFilename); + if( zLockFile==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); + } + pNew->lockingContext = zLockFile; + } -/* -** Sleep for a little while. Return the amount of time slept. -** The argument is the number of microseconds we want to sleep. -** The return value is the number of microseconds of sleep actually -** requested from the underlying operating system, a number which -** might be greater than or equal to the argument, but not less -** than the argument. -*/ -static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ #if OS_VXWORKS - struct timespec sp; - - sp.tv_sec = microseconds / 1000000; - sp.tv_nsec = (microseconds % 1000000) * 1000; - nanosleep(&sp, NULL); - UNUSED_PARAMETER(NotUsed); - return microseconds; -#elif defined(HAVE_USLEEP) && HAVE_USLEEP - usleep(microseconds); - UNUSED_PARAMETER(NotUsed); - return microseconds; -#else - int seconds = (microseconds+999999)/1000000; - sleep(seconds); - UNUSED_PARAMETER(NotUsed); - return seconds*1000000; + else if( pLockingStyle == &semIoMethods ){ + /* Named semaphore locking uses the file path so it needs to be + ** included in the semLockingContext + */ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( (rc==SQLITE_OK) && (pNew->pInode->pSem==NULL) ){ + char *zSemName = pNew->pInode->aSemName; + int n; + sqlite3_snprintf(MAX_PATHNAME, zSemName, "/%s.sem", + pNew->pId->zCanonicalName); + for( n=1; zSemName[n]; n++ ) + if( zSemName[n]=='/' ) zSemName[n] = '_'; + pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); + if( pNew->pInode->pSem == SEM_FAILED ){ + rc = SQLITE_NOMEM; + pNew->pInode->aSemName[0] = '\0'; + } + } + unixLeaveMutex(); + } +#endif + + pNew->lastErrno = 0; +#if OS_VXWORKS + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + h = -1; + unlink(zFilename); + isDelete = 0; + } + pNew->isDelete = isDelete; #endif + if( rc!=SQLITE_OK ){ + if( dirfd>=0 ) robust_close(pNew, dirfd, __LINE__); + if( h>=0 ) robust_close(pNew, h, __LINE__); + }else{ + pNew->pMethod = pLockingStyle; + OpenCounter(+1); + } + return rc; } /* -** The following variable, if set to a non-zero value, is interpreted as -** the number of seconds since 1970 and is used to set the result of -** sqlite3OsCurrentTime() during testing. +** Open a file descriptor to the directory containing file zFilename. +** If successful, *pFd is set to the opened file descriptor and +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined +** value. +** +** If SQLITE_OK is returned, the caller is responsible for closing +** the file descriptor *pFd using close(). */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +static int openDirectory(const char *zFilename, int *pFd){ + int ii; + int fd = -1; + char zDirname[MAX_PATHNAME+1]; + + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); + for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); + if( ii>0 ){ + zDirname[ii] = '\0'; + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ +#ifdef FD_CLOEXEC + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); #endif + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); + } + } + *pFd = fd; + return (fd>=0?SQLITE_OK:unixLogError(SQLITE_CANTOPEN_BKPT, "open", zDirname)); +} /* -** Find the current time (in Universal Coordinated Time). Write into *piNow -** the current time and date as a Julian Day number times 86_400_000. In -** other words, write into *piNow the number of milliseconds since the Julian -** epoch of noon in Greenwich on November 24, 4714 B.C according to the -** proleptic Gregorian calendar. -** -** On success, return 0. Return 1 if the time and date cannot be found. +** Return the name of a directory in which to put temporary files. +** If no suitable temporary file directory can be found, return NULL. */ -static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ - static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; -#if defined(NO_GETTOD) - time_t t; - time(&t); - *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; -#elif OS_VXWORKS - struct timespec sNow; - clock_gettime(CLOCK_REALTIME, &sNow); - *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; -#else - struct timeval sNow; - gettimeofday(&sNow, 0); - *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; -#endif +static const char *unixTempFileDir(void){ + static const char *azDirs[] = { + 0, + 0, + "/var/tmp", + "/usr/tmp", + "/tmp", + 0 /* List terminator */ + }; + unsigned int i; + struct stat buf; + const char *zDir = 0; -#ifdef SQLITE_TEST - if( sqlite3_current_time ){ - *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + azDirs[0] = sqlite3_temp_directory; + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); + for(i=0; imxPathname bytes. */ -static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ - sqlite3_int64 i; - UNUSED_PARAMETER(NotUsed); - unixCurrentTimeInt64(0, &i); - *prNow = i/86400000.0; - return 0; +static int unixGetTempname(int nBuf, char *zBuf){ + static const unsigned char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + unsigned int i, j; + const char *zDir; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. + */ + SimulateIOError( return SQLITE_IOERR ); + + zDir = unixTempFileDir(); + if( zDir==0 ) zDir = "."; + + /* Check that the output buffer is large enough for the temporary file + ** name. If it is not, return SQLITE_ERROR. + */ + if( (strlen(zDir) + strlen(SQLITE_TEMP_FILE_PREFIX) + 17) >= (size_t)nBuf ){ + return SQLITE_ERROR; + } + + do{ + sqlite3_snprintf(nBuf-17, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); + j = (int)strlen(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + }while( osAccess(zBuf,0)==0 ); + return SQLITE_OK; } +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) /* -** We added the xGetLastError() method with the intention of providing -** better low-level error messages when operating-system problems come up -** during SQLite operation. But so far, none of that has been implemented -** in the core. So this routine is never called. For now, it is merely -** a place-holder. +** Routine to transform a unixFile into a proxy-locking unixFile. +** Implementation in the proxy-lock division, but used by unixOpen() +** if SQLITE_PREFER_PROXY_LOCKING is defined. */ -static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ - UNUSED_PARAMETER(NotUsed); - UNUSED_PARAMETER(NotUsed2); - UNUSED_PARAMETER(NotUsed3); - return 0; -} - +static int proxyTransformUnixFile(unixFile*, const char*); +#endif /* -************************ End of sqlite3_vfs methods *************************** -******************************************************************************/ - -/****************************************************************************** -************************** Begin Proxy Locking ******************************** -** -** Proxy locking is a "uber-locking-method" in this sense: It uses the -** other locking methods on secondary lock files. Proxy locking is a -** meta-layer over top of the primitive locking implemented above. For -** this reason, the division that implements of proxy locking is deferred -** until late in the file (here) after all of the other I/O methods have -** been defined - so that the primitive locking methods are available -** as services to help with the implementation of proxy locking. -** -**** -** -** The default locking schemes in SQLite use byte-range locks on the -** database file to coordinate safe, concurrent access by multiple readers -** and writers [http://sqlite.org/lockingv3.html]. The five file locking -** states (UNLOCKED, PENDING, SHARED, RESERVED, EXCLUSIVE) are implemented -** as POSIX read & write locks over fixed set of locations (via fsctl), -** on AFP and SMB only exclusive byte-range locks are available via fsctl -** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. -** To simulate a F_RDLCK on the shared range, on AFP a randomly selected -** address in the shared range is taken for a SHARED lock, the entire -** shared range is taken for an EXCLUSIVE lock): -** -** PENDING_BYTE 0x40000000 -** RESERVED_BYTE 0x40000001 -** SHARED_RANGE 0x40000002 -> 0x40000200 -** -** This works well on the local file system, but shows a nearly 100x -** slowdown in read performance on AFP because the AFP client disables -** the read cache when byte-range locks are present. Enabling the read -** cache exposes a cache coherency problem that is present on all OS X -** supported network file systems. NFS and AFP both observe the -** close-to-open semantics for ensuring cache coherency -** [http://nfs.sourceforge.net/#faq_a8], which does not effectively -** address the requirements for concurrent database access by multiple -** readers and writers -** [http://www.nabble.com/SQLite-on-NFS-cache-coherency-td15655701.html]. +** Search for an unused file descriptor that was opened on the database +** file (not a journal or master-journal file) identified by pathname +** zPath with SQLITE_OPEN_XXX flags matching those passed as the second +** argument to this function. ** -** To address the performance and cache coherency issues, proxy file locking -** changes the way database access is controlled by limiting access to a -** single host at a time and moving file locks off of the database file -** and onto a proxy file on the local file system. +** Such a file descriptor may exist if a database connection was closed +** but the associated file descriptor could not be closed because some +** other file descriptor open on the same file is holding a file-lock. +** Refer to comments in the unixClose() function and the lengthy comment +** describing "Posix Advisory Locking" at the start of this file for +** further details. Also, ticket #4018. ** +** If a suitable file descriptor is found, then it is returned. If no +** such file descriptor is located, -1 is returned. +*/ +static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ + UnixUnusedFd *pUnused = 0; + + /* Do not search for an unused file descriptor on vxworks. Not because + ** vxworks would not benefit from the change (it might, we're not sure), + ** but because no way to test it is currently available. It is better + ** not to risk breaking vxworks support for the sake of such an obscure + ** feature. */ +#if !OS_VXWORKS + struct stat sStat; /* Results of stat() call */ + + /* A stat() call may fail for various reasons. If this happens, it is + ** almost certain that an open() call on the same path will also fail. + ** For this reason, if an error occurs in the stat() call here, it is + ** ignored and -1 is returned. The caller will try to open a new file + ** descriptor on the same path, fail, and return an error to SQLite. + ** + ** Even if a subsequent open() call does succeed, the consequences of + ** not searching for a resusable file descriptor are not dire. */ + if( 0==stat(zPath, &sStat) ){ + unixInodeInfo *pInode; + + unixEnterMutex(); + pInode = inodeList; + while( pInode && (pInode->fileId.dev!=sStat.st_dev + || pInode->fileId.ino!=sStat.st_ino) ){ + pInode = pInode->pNext; + } + if( pInode ){ + UnixUnusedFd **pp; + for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); + pUnused = *pp; + if( pUnused ){ + *pp = pUnused->pNext; + } + } + unixLeaveMutex(); + } +#endif /* if !OS_VXWORKS */ + return pUnused; +} + +/* +** This function is called by unixOpen() to determine the unix permissions +** to create new files with. If no error occurs, then SQLITE_OK is returned +** and a value suitable for passing as the third argument to open(2) is +** written to *pMode. If an IO error occurs, an SQLite error code is +** returned and the value of *pMode is not modified. ** -** Using proxy locks -** ----------------- +** If the file being opened is a temporary file, it is always created with +** the octal permissions 0600 (read/writable by owner only). If the file +** is a database or master journal file, it is created with the permissions +** mask SQLITE_DEFAULT_FILE_PERMISSIONS. ** -** C APIs +** Finally, if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions +** as the associated database file. ** -** sqlite3_file_control(db, dbname, SQLITE_SET_LOCKPROXYFILE, -** | ":auto:"); -** sqlite3_file_control(db, dbname, SQLITE_GET_LOCKPROXYFILE, &); +** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the +** original filename is unavailable. But 8_3_NAMES is only used for +** FAT filesystems and permissions do not matter there, so just use +** the default permissions. +*/ +static int findCreateFileMode( + const char *zPath, /* Path of file (possibly) being created */ + int flags, /* Flags passed as 4th argument to xOpen() */ + mode_t *pMode /* OUT: Permissions to open file with */ +){ + int rc = SQLITE_OK; /* Return Code */ + *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + char zDb[MAX_PATHNAME+1]; /* Database file path */ + int nDb; /* Number of valid bytes in zDb */ + struct stat sStat; /* Output of stat() on database file */ + + /* zPath is a path to a WAL or journal file. The following block derives + ** the path to the associated database file from zPath. This block handles + ** the following naming conventions: + ** + ** "-journal" + ** "-wal" + ** "-journalNN" + ** "-walNN" + ** + ** where NN is a 4 digit decimal number. The NN naming schemes are + ** used by the test_multiplex.c module. + */ + nDb = sqlite3Strlen30(zPath) - 1; + while( nDb>0 && zPath[nDb]!='-' ) nDb--; + if( nDb==0 ) return SQLITE_OK; + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + + if( 0==stat(zDb, &sStat) ){ + *pMode = sStat.st_mode & 0777; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + *pMode = 0600; + } + return rc; +} + +/* +** Open the file zPath. +** +** Previously, the SQLite OS layer used three functions in place of this +** one: ** +** sqlite3OsOpenReadWrite(); +** sqlite3OsOpenReadOnly(); +** sqlite3OsOpenExclusive(); ** -** SQL pragmas +** These calls correspond to the following combinations of flags: ** -** PRAGMA [database.]lock_proxy_file= | :auto: -** PRAGMA [database.]lock_proxy_file +** ReadWrite() -> (READWRITE | CREATE) +** ReadOnly() -> (READONLY) +** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) ** -** Specifying ":auto:" means that if there is a conch file with a matching -** host ID in it, the proxy path in the conch file will be used, otherwise -** a proxy path based on the user's temp dir -** (via confstr(_CS_DARWIN_USER_TEMP_DIR,...)) will be used and the -** actual proxy file name is generated from the name and path of the -** database file. For example: -** -** For database path "/Users/me/foo.db" -** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") -** -** Once a lock proxy is configured for a database connection, it can not -** be removed, however it may be switched to a different proxy path via -** the above APIs (assuming the conch file is not being held by another -** connection or process). -** -** -** How proxy locking works -** ----------------------- -** -** Proxy file locking relies primarily on two new supporting files: -** -** * conch file to limit access to the database file to a single host -** at a time -** -** * proxy file to act as a proxy for the advisory locks normally -** taken on the database -** -** The conch file - to use a proxy file, sqlite must first "hold the conch" -** by taking an sqlite-style shared lock on the conch file, reading the -** contents and comparing the host's unique host ID (see below) and lock -** proxy path against the values stored in the conch. The conch file is -** stored in the same directory as the database file and the file name -** is patterned after the database file name as ".-conch". -** If the conch file does not exist, or it's contents do not match the -** host ID and/or proxy path, then the lock is escalated to an exclusive -** lock and the conch file contents is updated with the host ID and proxy -** path and the lock is downgraded to a shared lock again. If the conch -** is held by another process (with a shared lock), the exclusive lock -** will fail and SQLITE_BUSY is returned. -** -** The proxy file - a single-byte file used for all advisory file locks -** normally taken on the database file. This allows for safe sharing -** of the database file for multiple readers and writers on the same -** host (the conch ensures that they all use the same local lock file). -** -** Requesting the lock proxy does not immediately take the conch, it is -** only taken when the first request to lock database file is made. -** This matches the semantics of the traditional locking behavior, where -** opening a connection to a database file does not take a lock on it. -** The shared lock and an open file descriptor are maintained until -** the connection to the database is closed. -** -** The proxy file and the lock file are never deleted so they only need -** to be created the first time they are used. -** -** Configuration options -** --------------------- -** -** SQLITE_PREFER_PROXY_LOCKING -** -** Database files accessed on non-local file systems are -** automatically configured for proxy locking, lock files are -** named automatically using the same logic as -** PRAGMA lock_proxy_file=":auto:" -** -** SQLITE_PROXY_DEBUG -** -** Enables the logging of error messages during host id file -** retrieval and creation -** -** LOCKPROXYDIR -** -** Overrides the default directory used for lock proxy files that -** are named automatically via the ":auto:" setting -** -** SQLITE_DEFAULT_PROXYDIR_PERMISSIONS -** -** Permissions to use when creating a directory for storing the -** lock proxy files, only used when LOCKPROXYDIR is not set. -** -** -** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, -** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will -** force proxy locking to be used for every database file opened, and 0 -** will force automatic proxy locking to be disabled for all database -** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or -** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). +** The old OpenExclusive() accepted a boolean argument - "delFlag". If +** true, the file was configured to be automatically deleted when the +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for +** OpenExclusive(). */ +static int unixOpen( + sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */ + const char *zPath, /* Pathname of file to be opened */ + sqlite3_file *pFile, /* The file descriptor to be filled in */ + int flags, /* Input flags to control the opening */ + int *pOutFlags /* Output flags returned to SQLite core */ +){ + unixFile *p = (unixFile *)pFile; + int fd = -1; /* File descriptor returned by open() */ + int dirfd = -1; /* Directory file descriptor */ + int openFlags = 0; /* Flags to pass to open() */ + int eType = flags&0xFFFFFF00; /* Type of file to open */ + int noLock; /* True to omit locking primitives */ + int rc = SQLITE_OK; /* Function Return Code */ -/* -** Proxy locking is only available on MacOSX -*/ -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +#if SQLITE_ENABLE_LOCKING_STYLE + int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); +#endif -/* -** The proxyLockingContext has the path and file structures for the remote -** and local proxy files in it -*/ -typedef struct proxyLockingContext proxyLockingContext; -struct proxyLockingContext { - unixFile *conchFile; /* Open conch file */ - char *conchFilePath; /* Name of the conch file */ - unixFile *lockProxy; /* Open proxy lock file */ - char *lockProxyPath; /* Name of the proxy lock file */ - char *dbPath; /* Name of the open file */ - int conchHeld; /* 1 if the conch is held, -1 if lockless */ - void *oldLockingContext; /* Original lockingcontext to restore on close */ - sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ -}; + /* If creating a master or main-file journal, this function will open + ** a file-descriptor on the directory too. The first time unixSync() + ** is called the directory file descriptor will be fsync()ed and close()d. + */ + int isOpenDirectory = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); -/* -** The proxy lock file path for the database at dbPath is written into lPath, -** which must point to valid, writable memory large enough for a maxLen length -** file path. -*/ -static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ - int len; - int dbLen; - int i; + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATHNAME+1]; + const char *zName = zPath; -#ifdef LOCKPROXYDIR - len = strlcpy(lPath, LOCKPROXYDIR, maxLen); -#else -# ifdef _CS_DARWIN_USER_TEMP_DIR - { - if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ - OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", - lPath, errno, getpid())); - return SQLITE_IOERR_LOCK; - } - len = strlcat(lPath, "sqliteplocks", maxLen); - } -# else - len = strlcpy(lPath, "/tmp/", maxLen); -# endif -#endif + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); - if( lPath[len-1]!='/' ){ - len = strlcat(lPath, "/", maxLen); - } - - /* transform the db path to a unique cache name */ - dbLen = (int)strlen(dbPath); - for( i=0; i 0) ){ - /* only mkdir if leaf dir != "." or "/" or ".." */ - if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') - || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ - buf[i]='\0'; - if( mkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ - int err=errno; - if( err!=EEXIST ) { - OSTRACE(("CREATELOCKPATH FAILED creating %s, " - "'%s' proxy lock path=%s pid=%d\n", - buf, strerror(err), lockPath, getpid())); - return err; - } - } + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + memset(p, 0, sizeof(unixFile)); + + if( eType==SQLITE_OPEN_MAIN_DB ){ + UnixUnusedFd *pUnused; + pUnused = findReusableFd(zName, flags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM; } - start=i+1; } - buf[i] = lockPath[i]; + p->pUnused = pUnused; + }else if( !zName ){ + /* If zName is NULL, the upper layer is requesting a temp file. */ + assert(isDelete && !isOpenDirectory); + rc = unixGetTempname(MAX_PATHNAME+1, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zName = zTmpname; } - OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid())); - return 0; -} -/* -** Create a new VFS file descriptor (stored in memory obtained from -** sqlite3_malloc) and open the file named "path" in the file descriptor. -** -** The caller is responsible not only for closing the file descriptor -** but also for freeing the memory associated with the file descriptor. -*/ -static int proxyCreateUnixFile( - const char *path, /* path for the new unixFile */ - unixFile **ppFile, /* unixFile created and returned by ref */ - int islockfile /* if non zero missing dirs will be created */ -) { - int fd = -1; - int dirfd = -1; - unixFile *pNew; - int rc = SQLITE_OK; - int openFlags = O_RDWR | O_CREAT; - sqlite3_vfs dummyVfs; - int terrno = 0; - UnixUnusedFd *pUnused = NULL; + /* Determine the value of the flags parameter passed to POSIX function + ** open(). These must be calculated even if open() is not called, as + ** they may be stored as part of the file handle and used by the + ** 'conch file' locking functions later on. */ + if( isReadonly ) openFlags |= O_RDONLY; + if( isReadWrite ) openFlags |= O_RDWR; + if( isCreate ) openFlags |= O_CREAT; + if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); + openFlags |= (O_LARGEFILE|O_BINARY); - /* 1. first try to open/create the file - ** 2. if that fails, and this is a lock file (not-conch), try creating - ** the parent directories and then try again. - ** 3. if that fails, try to open the file read-only - ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file - */ - pUnused = findReusableFd(path, openFlags); - if( pUnused ){ - fd = pUnused->fd; - }else{ - pUnused = sqlite3_malloc(sizeof(*pUnused)); - if( !pUnused ){ - return SQLITE_NOMEM; - } - } if( fd<0 ){ - fd = open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); - terrno = errno; - if( fd<0 && errno==ENOENT && islockfile ){ - if( proxyCreateLockPath(path) == SQLITE_OK ){ - fd = open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); - } + mode_t openMode; /* Permissions to create file with */ + rc = findCreateFileMode(zName, flags, &openMode); + if( rc!=SQLITE_OK ){ + assert( !p->pUnused ); + assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); + return rc; } - } - if( fd<0 ){ - openFlags = O_RDONLY; - fd = open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); - terrno = errno; - } - if( fd<0 ){ - if( islockfile ){ - return SQLITE_BUSY; + fd = robust_open(zName, openFlags, openMode); + OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); + if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ + /* Failed to open the file for read/write access. Try read-only. */ + flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + openFlags &= ~(O_RDWR|O_CREAT); + flags |= SQLITE_OPEN_READONLY; + openFlags |= O_RDONLY; + isReadonly = 1; + fd = robust_open(zName, openFlags, openMode); } - switch (terrno) { - case EACCES: - return SQLITE_PERM; - case EIO: - return SQLITE_IOERR_LOCK; /* even though it is the conch */ - default: - return SQLITE_CANTOPEN_BKPT; + if( fd<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + goto open_finished; } } - - pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); - if( pNew==NULL ){ - rc = SQLITE_NOMEM; - goto end_create_proxy; + assert( fd>=0 ); + if( pOutFlags ){ + *pOutFlags = flags; } - memset(pNew, 0, sizeof(unixFile)); - pNew->openFlags = openFlags; - dummyVfs.pAppData = (void*)&autolockIoFinder; - pUnused->fd = fd; - pUnused->flags = openFlags; - pNew->pUnused = pUnused; - - rc = fillInUnixFile(&dummyVfs, fd, dirfd, (sqlite3_file*)pNew, path, 0, 0); - if( rc==SQLITE_OK ){ - *ppFile = pNew; - return SQLITE_OK; + + if( p->pUnused ){ + p->pUnused->fd = fd; + p->pUnused->flags = flags; } -end_create_proxy: - close(fd); /* silently leak fd if error, we're already in error */ - sqlite3_free(pNew); - sqlite3_free(pUnused); - return rc; -} -#ifdef SQLITE_TEST -/* simulate multiple hosts by creating unique hostid file paths */ -SQLITE_API int sqlite3_hostid_num = 0; + if( isDelete ){ +#if OS_VXWORKS + zPath = zName; +#else + unlink(zName); +#endif + } +#if SQLITE_ENABLE_LOCKING_STYLE + else{ + p->openFlags = openFlags; + } #endif -#define PROXY_HOSTIDLEN 16 /* conch file host id length */ + if( isOpenDirectory ){ + rc = openDirectory(zPath, &dirfd); + if( rc!=SQLITE_OK ){ + /* It is safe to close fd at this point, because it is guaranteed not + ** to be open on a database file. If it were open on a database file, + ** it would not be safe to close as this would release any locks held + ** on the file by this process. */ + assert( eType!=SQLITE_OPEN_MAIN_DB ); + robust_close(p, fd, __LINE__); + goto open_finished; + } + } -/* Not always defined in the headers as it ought to be */ -extern int gethostuuid(uuid_t id, const struct timespec *wait); +#ifdef FD_CLOEXEC + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + + noLock = eType!=SQLITE_OPEN_MAIN_DB; -/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN -** bytes of writable memory. -*/ -static int proxyGetHostID(unsigned char *pHostID, int *pError){ - struct timespec timeout = {1, 0}; /* 1 sec timeout */ - assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); - memset(pHostID, 0, PROXY_HOSTIDLEN); -#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ - && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 - if( gethostuuid(pHostID, &timeout) ){ - int err = errno; - if( pError ){ - *pError = err; - } - return SQLITE_IOERR; +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; + if( fstatfs(fd, &fsInfo) == -1 ){ + ((unixFile*)pFile)->lastErrno = errno; + if( dirfd>=0 ) robust_close(p, dirfd, __LINE__); + robust_close(p, fd, __LINE__); + return SQLITE_IOERR_ACCESS; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; } #endif -#ifdef SQLITE_TEST - /* simulate multiple hosts by creating unique hostid file paths */ - if( sqlite3_hostid_num != 0){ - pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); + +#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_PREFER_PROXY_LOCKING + isAutoProxy = 1; +#endif + if( isAutoProxy && (zPath!=NULL) && (!noLock) && pVfs->xOpen ){ + char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); + int useProxy = 0; + + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means + ** never use proxy, NULL means use proxy for non-local files only. */ + if( envforce!=NULL ){ + useProxy = atoi(envforce)>0; + }else{ + struct statfs fsInfo; + if( statfs(zPath, &fsInfo) == -1 ){ + /* In theory, the close(fd) call is sub-optimal. If the file opened + ** with fd is a database file, and there are other connections open + ** on that file that are currently holding advisory locks on it, + ** then the call to close() will cancel those locks. In practice, + ** we're assuming that statfs() doesn't fail very often. At least + ** not while other file descriptors opened by the same process on + ** the same file are working. */ + p->lastErrno = errno; + if( dirfd>=0 ){ + robust_close(p, dirfd, __LINE__); + } + robust_close(p, fd, __LINE__); + rc = SQLITE_IOERR_ACCESS; + goto open_finished; + } + useProxy = !(fsInfo.f_flags&MNT_LOCAL); + } + if( useProxy ){ + rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, + isDelete, isReadonly); + if( rc==SQLITE_OK ){ + rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); + if( rc!=SQLITE_OK ){ + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + */ + unixClose(pFile); + return rc; + } + } + goto open_finished; + } } #endif - return SQLITE_OK; + rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, + isDelete, isReadonly); +open_finished: + if( rc!=SQLITE_OK ){ + sqlite3_free(p->pUnused); + } + return rc; } -/* The conch file contains the header, host id and lock file path - */ -#define PROXY_CONCHVERSION 2 /* 1-byte header, 16-byte host id, path */ -#define PROXY_HEADERLEN 1 /* conch file header length */ -#define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) -#define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) -/* -** Takes an open conch file, copies the contents to a new path and then moves -** it back. The newly created file's file descriptor is assigned to the -** conch file structure and finally the original conch file descriptor is -** closed. Returns zero if successful. +/* +** Delete the file at zPath. If the dirSync argument is true, fsync() +** the directory after deleting the file. */ -static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - unixFile *conchFile = pCtx->conchFile; - char tPath[MAXPATHLEN]; - char buf[PROXY_MAXCONCHLEN]; - char *cPath = pCtx->conchFilePath; - size_t readLen = 0; - size_t pathLen = 0; - char errmsg[64] = ""; - int fd = -1; - int rc = -1; - UNUSED_PARAMETER(myHostID); - - /* create a new path by replace the trailing '-conch' with '-break' */ - pathLen = strlcpy(tPath, cPath, MAXPATHLEN); - if( pathLen>MAXPATHLEN || pathLen<6 || - (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ - sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); - goto end_breaklock; +static int unixDelete( + sqlite3_vfs *NotUsed, /* VFS containing this as the xDelete method */ + const char *zPath, /* Name of file to be deleted */ + int dirSync /* If true, fsync() directory after deleting file */ +){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(NotUsed); + SimulateIOError(return SQLITE_IOERR_DELETE); + if( unlink(zPath)==(-1) && errno!=ENOENT ){ + return unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); } - /* read the conch content */ - readLen = pread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); - if( readLenh); - conchFile->h = fd; - conchFile->openFlags = O_RDWR | O_CREAT; - -end_breaklock: - if( rc ){ - if( fd>=0 ){ - unlink(tPath); - close(fd); - } - fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); - } - return rc; -} - -/* Take the requested lock on the conch file and break a stale lock if the -** host id matches. -*/ -static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - unixFile *conchFile = pCtx->conchFile; - int rc = SQLITE_OK; - int nTries = 0; - struct timespec conchModTime; - - do { - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); - nTries ++; - if( rc==SQLITE_BUSY ){ - /* If the lock failed (busy): - * 1st try: get the mod time of the conch, wait 0.5s and try again. - * 2nd try: fail if the mod time changed or host id is different, wait - * 10 sec and try again - * 3rd try: break the lock unless the mod time has changed. - */ - struct stat buf; - if( fstat(conchFile->h, &buf) ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_LOCK; - } - - if( nTries==1 ){ - conchModTime = buf.st_mtimespec; - usleep(500000); /* wait 0.5 sec and try the lock again*/ - continue; - } - - assert( nTries>1 ); - if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || - conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ - return SQLITE_BUSY; - } - - if( nTries==2 ){ - char tBuf[PROXY_MAXCONCHLEN]; - int len = pread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); - if( len<0 ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_LOCK; - } - if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ - /* don't break the lock if the host id doesn't match */ - if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ - return SQLITE_BUSY; - } - }else{ - /* don't break the lock on short read or a version mismatch */ - return SQLITE_BUSY; - } - usleep(10000000); /* wait 10 sec and try the lock again */ - continue; - } - - assert( nTries==3 ); - if( 0==proxyBreakConchLock(pFile, myHostID) ){ - rc = SQLITE_OK; - if( lockType==EXCLUSIVE_LOCK ){ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); - } - if( !rc ){ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); - } - } - } - } while( rc==SQLITE_BUSY && nTries<3 ); - - return rc; -} - -/* Takes the conch by taking a shared lock and read the contents conch, if -** lockPath is non-NULL, the host ID and lock file path must match. A NULL -** lockPath means that the lockPath in the conch file will be used if the -** host IDs match, or a new lock path will be generated automatically -** and written to the conch file. -*/ -static int proxyTakeConch(unixFile *pFile){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - - if( pCtx->conchHeld!=0 ){ - return SQLITE_OK; - }else{ - unixFile *conchFile = pCtx->conchFile; - uuid_t myHostID; - int pError = 0; - char readBuf[PROXY_MAXCONCHLEN]; - char lockPath[MAXPATHLEN]; - char *tempLockPath = NULL; - int rc = SQLITE_OK; - int createConch = 0; - int hostIdMatch = 0; - int readLen = 0; - int tryOldLockPath = 0; - int forceNewLockPath = 0; - - OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, - (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid())); - - rc = proxyGetHostID(myHostID, &pError); - if( (rc&0xff)==SQLITE_IOERR ){ - pFile->lastErrno = pError; - goto end_takeconch; - } - rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - goto end_takeconch; - } - /* read the existing conch file */ - readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); - if( readLen<0 ){ - /* I/O error: lastErrno set by seekAndRead */ - pFile->lastErrno = conchFile->lastErrno; - rc = SQLITE_IOERR_READ; - goto end_takeconch; - }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || - readBuf[0]!=(char)PROXY_CONCHVERSION ){ - /* a short read or version format mismatch means we need to create a new - ** conch file. - */ - createConch = 1; - } - /* if the host id matches and the lock path already exists in the conch - ** we'll try to use the path there, if we can't open that path, we'll - ** retry with a new auto-generated path - */ - do { /* in case we need to try again for an :auto: named lock file */ - - if( !createConch && !forceNewLockPath ){ - hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, - PROXY_HOSTIDLEN); - /* if the conch has data compare the contents */ - if( !pCtx->lockProxyPath ){ - /* for auto-named local lock file, just check the host ID and we'll - ** use the local lock file path that's already in there - */ - if( hostIdMatch ){ - size_t pathLen = (readLen - PROXY_PATHINDEX); - - if( pathLen>=MAXPATHLEN ){ - pathLen=MAXPATHLEN-1; - } - memcpy(lockPath, &readBuf[PROXY_PATHINDEX], pathLen); - lockPath[pathLen] = 0; - tempLockPath = lockPath; - tryOldLockPath = 1; - /* create a copy of the lock path if the conch is taken */ - goto end_takeconch; - } - }else if( hostIdMatch - && !strncmp(pCtx->lockProxyPath, &readBuf[PROXY_PATHINDEX], - readLen-PROXY_PATHINDEX) - ){ - /* conch host and lock path match */ - goto end_takeconch; - } - } - - /* if the conch isn't writable and doesn't match, we can't take it */ - if( (conchFile->openFlags&O_RDWR) == 0 ){ - rc = SQLITE_BUSY; - goto end_takeconch; - } - - /* either the conch didn't match or we need to create a new one */ - if( !pCtx->lockProxyPath ){ - proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); - tempLockPath = lockPath; - /* create a copy of the lock path _only_ if the conch is taken */ - } - - /* update conch with host and path (this will fail if other process - ** has a shared lock already), if the host id matches, use the big - ** stick. - */ - futimes(conchFile->h, NULL); - if( hostIdMatch && !createConch ){ - if( conchFile->pInode && conchFile->pInode->nShared>1 ){ - /* We are trying for an exclusive lock but another thread in this - ** same process is still holding a shared lock. */ - rc = SQLITE_BUSY; - } else { - rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); - } - }else{ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); - } - if( rc==SQLITE_OK ){ - char writeBuffer[PROXY_MAXCONCHLEN]; - int writeSize = 0; - - writeBuffer[0] = (char)PROXY_CONCHVERSION; - memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); - if( pCtx->lockProxyPath!=NULL ){ - strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); - }else{ - strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); - } - writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); - ftruncate(conchFile->h, writeSize); - rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); - fsync(conchFile->h); - /* If we created a new conch file (not just updated the contents of a - ** valid conch file), try to match the permissions of the database - */ - if( rc==SQLITE_OK && createConch ){ - struct stat buf; - int err = fstat(pFile->h, &buf); - if( err==0 ){ - mode_t cmode = buf.st_mode&(S_IRUSR|S_IWUSR | S_IRGRP|S_IWGRP | - S_IROTH|S_IWOTH); - /* try to match the database file R/W permissions, ignore failure */ -#ifndef SQLITE_PROXY_DEBUG - fchmod(conchFile->h, cmode); -#else - if( fchmod(conchFile->h, cmode)!=0 ){ - int code = errno; - fprintf(stderr, "fchmod %o FAILED with %d %s\n", - cmode, code, strerror(code)); - } else { - fprintf(stderr, "fchmod %o SUCCEDED\n",cmode); - } - }else{ - int code = errno; - fprintf(stderr, "STAT FAILED[%d] with %d %s\n", - err, code, strerror(code)); -#endif - } - } - } - conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); - - end_takeconch: - OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); - if( rc==SQLITE_OK && pFile->openFlags ){ - if( pFile->h>=0 ){ -#ifdef STRICT_CLOSE_ERROR - if( close(pFile->h) ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_CLOSE; - } +#ifndef SQLITE_DISABLE_DIRSYNC + if( dirSync ){ + int fd; + rc = openDirectory(zPath, &fd); + if( rc==SQLITE_OK ){ +#if OS_VXWORKS + if( fsync(fd)==-1 ) #else - close(pFile->h); /* silently leak fd if fail */ + if( fsync(fd) ) #endif - } - pFile->h = -1; - int fd = open(pCtx->dbPath, pFile->openFlags, - SQLITE_DEFAULT_FILE_PERMISSIONS); - OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); - if( fd>=0 ){ - pFile->h = fd; - }else{ - rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called - during locking */ - } - } - if( rc==SQLITE_OK && !pCtx->lockProxy ){ - char *path = tempLockPath ? tempLockPath : pCtx->lockProxyPath; - rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); - if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ - /* we couldn't create the proxy lock file with the old lock file path - ** so try again via auto-naming - */ - forceNewLockPath = 1; - tryOldLockPath = 0; - continue; /* go back to the do {} while start point, try again */ - } - } - if( rc==SQLITE_OK ){ - /* Need to make a copy of path if we extracted the value - ** from the conch file or the path was allocated on the stack - */ - if( tempLockPath ){ - pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); - if( !pCtx->lockProxyPath ){ - rc = SQLITE_NOMEM; - } - } - } - if( rc==SQLITE_OK ){ - pCtx->conchHeld = 1; - - if( pCtx->lockProxy->pMethod == &afpIoMethods ){ - afpLockingContext *afpCtx; - afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; - afpCtx->dbPath = pCtx->lockProxyPath; - } - } else { - conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + { + rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } - OSTRACE(("TAKECONCH %d %s\n", conchFile->h, - rc==SQLITE_OK?"ok":"failed")); - return rc; - } while (1); /* in case we need to retry the :auto: lock file - - ** we should never get here except via the 'continue' call. */ - } -} - -/* -** If pFile holds a lock on a conch file, then release that lock. -*/ -static int proxyReleaseConch(unixFile *pFile){ - int rc = SQLITE_OK; /* Subroutine return code */ - proxyLockingContext *pCtx; /* The locking context for the proxy lock */ - unixFile *conchFile; /* Name of the conch file */ - - pCtx = (proxyLockingContext *)pFile->lockingContext; - conchFile = pCtx->conchFile; - OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, - (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), - getpid())); - if( pCtx->conchHeld>0 ){ - rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + robust_close(0, fd, __LINE__); + } } - pCtx->conchHeld = 0; - OSTRACE(("RELEASECONCH %d %s\n", conchFile->h, - (rc==SQLITE_OK ? "ok" : "failed"))); +#endif return rc; } /* -** Given the name of a database file, compute the name of its conch file. -** Store the conch filename in memory obtained from sqlite3_malloc(). -** Make *pConchPath point to the new name. Return SQLITE_OK on success -** or SQLITE_NOMEM if unable to obtain memory. +** Test the existance of or access permissions of file zPath. The +** test performed depends on the value of flags: ** -** The caller is responsible for ensuring that the allocated memory -** space is eventually freed. +** SQLITE_ACCESS_EXISTS: Return 1 if the file exists +** SQLITE_ACCESS_READWRITE: Return 1 if the file is read and writable. +** SQLITE_ACCESS_READONLY: Return 1 if the file is readable. ** -** *pConchPath is set to NULL if a memory allocation error occurs. +** Otherwise return 0. */ -static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ - int i; /* Loop counter */ - int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ - char *conchPath; /* buffer in which to construct conch name */ +static int unixAccess( + sqlite3_vfs *NotUsed, /* The VFS containing this xAccess method */ + const char *zPath, /* Path of the file to examine */ + int flags, /* What do we want to learn about the zPath file? */ + int *pResOut /* Write result boolean here */ +){ + int amode = 0; + UNUSED_PARAMETER(NotUsed); + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + switch( flags ){ + case SQLITE_ACCESS_EXISTS: + amode = F_OK; + break; + case SQLITE_ACCESS_READWRITE: + amode = W_OK|R_OK; + break; + case SQLITE_ACCESS_READ: + amode = R_OK; + break; - /* Allocate space for the conch filename and initialize the name to - ** the name of the original database file. */ - *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); - if( conchPath==0 ){ - return SQLITE_NOMEM; + default: + assert(!"Invalid flags argument"); } - memcpy(conchPath, dbPath, len+1); - - /* now insert a "." before the last / character */ - for( i=(len-1); i>=0; i-- ){ - if( conchPath[i]=='/' ){ - i++; - break; + *pResOut = (osAccess(zPath, amode)==0); + if( flags==SQLITE_ACCESS_EXISTS && *pResOut ){ + struct stat buf; + if( 0==stat(zPath, &buf) && buf.st_size==0 ){ + *pResOut = 0; } } - conchPath[i]='.'; - while ( ilockingContext; - char *oldPath = pCtx->lockProxyPath; - int rc = SQLITE_OK; +static int unixFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zPath, /* Possibly relative input path */ + int nOut, /* Size of output buffer in bytes */ + char *zOut /* Output buffer */ +){ - if( pFile->eFileLock!=NO_LOCK ){ - return SQLITE_BUSY; - } + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); - /* nothing to do if the path is NULL, :auto: or matches the existing path */ - if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || - (oldPath && !strncmp(oldPath, path, MAXPATHLEN)) ){ - return SQLITE_OK; + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); + + zOut[nOut-1] = '\0'; + if( zPath[0]=='/' ){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); }else{ - unixFile *lockProxy = pCtx->lockProxy; - pCtx->lockProxy=NULL; - pCtx->conchHeld = 0; - if( lockProxy!=NULL ){ - rc=lockProxy->pMethod->xClose((sqlite3_file *)lockProxy); - if( rc ) return rc; - sqlite3_free(lockProxy); + int nCwd; + if( osGetcwd(zOut, nOut-1)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); } - sqlite3_free(oldPath); - pCtx->lockProxyPath = sqlite3DbStrDup(0, path); + nCwd = (int)strlen(zOut); + sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); } - - return rc; + return SQLITE_OK; } + +#ifndef SQLITE_OMIT_LOAD_EXTENSION /* -** pFile is a file that has been opened by a prior xOpen call. dbPath -** is a string buffer at least MAXPATHLEN+1 characters in size. -** -** This routine find the filename associated with pFile and writes it -** int dbPath. +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. */ -static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ -#if defined(__APPLE__) - if( pFile->pMethod == &afpIoMethods ){ - /* afp style keeps a reference to the db path in the filePath field - ** of the struct */ - assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); - strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, MAXPATHLEN); - } else -#endif - if( pFile->pMethod == &dotlockIoMethods ){ - /* dot lock style uses the locking context to store the dot lock - ** file path */ - int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); - memcpy(dbPath, (char *)pFile->lockingContext, len + 1); - }else{ - /* all other styles use the locking context to store the db file path */ - assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); - strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); - } - return SQLITE_OK; +#include +static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ + UNUSED_PARAMETER(NotUsed); + return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); } /* -** Takes an already filled in unix file and alters it so all file locking -** will be performed on the local proxy lock file. The following fields -** are preserved in the locking context so that they can be restored and -** the unix structure properly cleaned up at close time: -** ->lockingContext -** ->pMethod +** SQLite calls this function immediately after a call to unixDlSym() or +** unixDlOpen() fails (returns a null pointer). If a more detailed error +** message is available, it is written to zBufOut. If no error message +** is available, zBufOut is left unmodified and SQLite uses a default +** error message. */ -static int proxyTransformUnixFile(unixFile *pFile, const char *path) { - proxyLockingContext *pCtx; - char dbPath[MAXPATHLEN+1]; /* Name of the database file */ - char *lockPath=NULL; - int rc = SQLITE_OK; - - if( pFile->eFileLock!=NO_LOCK ){ - return SQLITE_BUSY; - } - proxyGetDbPathForUnixFile(pFile, dbPath); - if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ - lockPath=NULL; - }else{ - lockPath=(char *)path; - } - - OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, - (lockPath ? lockPath : ":auto:"), getpid())); - - pCtx = sqlite3_malloc( sizeof(*pCtx) ); - if( pCtx==0 ){ - return SQLITE_NOMEM; - } - memset(pCtx, 0, sizeof(*pCtx)); - - rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); - if( rc==SQLITE_OK ){ - rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile, 0); - if( rc==SQLITE_CANTOPEN && ((pFile->openFlags&O_RDWR) == 0) ){ - /* if (a) the open flags are not O_RDWR, (b) the conch isn't there, and - ** (c) the file system is read-only, then enable no-locking access. - ** Ugh, since O_RDONLY==0x0000 we test for !O_RDWR since unixOpen asserts - ** that openFlags will have only one of O_RDONLY or O_RDWR. - */ - struct statfs fsInfo; - struct stat conchInfo; - int goLockless = 0; - - if( stat(pCtx->conchFilePath, &conchInfo) == -1 ) { - int err = errno; - if( (err==ENOENT) && (statfs(dbPath, &fsInfo) != -1) ){ - goLockless = (fsInfo.f_flags&MNT_RDONLY) == MNT_RDONLY; - } - } - if( goLockless ){ - pCtx->conchHeld = -1; /* read only FS/ lockless */ - rc = SQLITE_OK; - } - } - } - if( rc==SQLITE_OK && lockPath ){ - pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); - } - - if( rc==SQLITE_OK ){ - pCtx->dbPath = sqlite3DbStrDup(0, dbPath); - if( pCtx->dbPath==NULL ){ - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - /* all memory is allocated, proxys are created and assigned, - ** switch the locking context and pMethod then return. - */ - pCtx->oldLockingContext = pFile->lockingContext; - pFile->lockingContext = pCtx; - pCtx->pOldMethod = pFile->pMethod; - pFile->pMethod = &proxyIoMethods; - }else{ - if( pCtx->conchFile ){ - pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); - sqlite3_free(pCtx->conchFile); - } - sqlite3DbFree(0, pCtx->lockProxyPath); - sqlite3_free(pCtx->conchFilePath); - sqlite3_free(pCtx); +static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ + const char *zErr; + UNUSED_PARAMETER(NotUsed); + unixEnterMutex(); + zErr = dlerror(); + if( zErr ){ + sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); } - OSTRACE(("TRANSPROXY %d %s\n", pFile->h, - (rc==SQLITE_OK ? "ok" : "failed"))); - return rc; + unixLeaveMutex(); } - +static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ + /* + ** GCC with -pedantic-errors says that C90 does not allow a void* to be + ** cast into a pointer to a function. And yet the library dlsym() routine + ** returns a void* which is really a pointer to a function. So how do we + ** use dlsym() with -pedantic-errors? + ** + ** Variable x below is defined to be a pointer to a function taking + ** parameters void* and const char* and returning a pointer to a function. + ** We initialize x by assigning it a pointer to the dlsym() function. + ** (That assignment requires a cast.) Then we call the function that + ** x points to. + ** + ** This work-around is unlikely to work correctly on any system where + ** you really cannot cast a function pointer into void*. But then, on the + ** other hand, dlsym() will not work on such a system either, so we have + ** not really lost anything. + */ + void (*(*x)(void*,const char*))(void); + UNUSED_PARAMETER(NotUsed); + x = (void(*(*)(void*,const char*))(void))dlsym; + return (*x)(p, zSym); +} +static void unixDlClose(sqlite3_vfs *NotUsed, void *pHandle){ + UNUSED_PARAMETER(NotUsed); + dlclose(pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define unixDlOpen 0 + #define unixDlError 0 + #define unixDlSym 0 + #define unixDlClose 0 +#endif /* -** This routine handles sqlite3_file_control() calls that are specific -** to proxy locking. +** Write nBuf bytes of random data to the supplied buffer zBuf. */ -static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ - switch( op ){ - case SQLITE_GET_LOCKPROXYFILE: { - unixFile *pFile = (unixFile*)id; - if( pFile->pMethod == &proxyIoMethods ){ - proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; - proxyTakeConch(pFile); - if( pCtx->lockProxyPath ){ - *(const char **)pArg = pCtx->lockProxyPath; - }else{ - *(const char **)pArg = ":auto: (not held)"; - } - } else { - *(const char **)pArg = NULL; - } - return SQLITE_OK; - } - case SQLITE_SET_LOCKPROXYFILE: { - unixFile *pFile = (unixFile*)id; - int rc = SQLITE_OK; - int isProxyStyle = (pFile->pMethod == &proxyIoMethods); - if( pArg==NULL || (const char *)pArg==0 ){ - if( isProxyStyle ){ - /* turn off proxy locking - not supported */ - rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; - }else{ - /* turn off proxy locking - already off - NOOP */ - rc = SQLITE_OK; - } - }else{ - const char *proxyPath = (const char *)pArg; - if( isProxyStyle ){ - proxyLockingContext *pCtx = - (proxyLockingContext*)pFile->lockingContext; - if( !strcmp(pArg, ":auto:") - || (pCtx->lockProxyPath && - !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) - ){ - rc = SQLITE_OK; - }else{ - rc = switchLockProxyPath(pFile, proxyPath); - } - }else{ - /* turn on proxy file locking */ - rc = proxyTransformUnixFile(pFile, proxyPath); - } - } - return rc; - } - default: { - assert( 0 ); /* The call assures that only valid opcodes are sent */ +static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ + UNUSED_PARAMETER(NotUsed); + assert((size_t)nBuf>=(sizeof(time_t)+sizeof(int))); + + /* We have to initialize zBuf to prevent valgrind from reporting + ** errors. The reports issued by valgrind are incorrect - we would + ** prefer that the randomness be increased by making use of the + ** uninitialized space in zBuf - but valgrind errors tend to worry + ** some users. Rather than argue, it seems easier just to initialize + ** the whole array and silence valgrind, even if that means less randomness + ** in the random seed. + ** + ** When testing, initializing zBuf[] to zero is all we do. That means + ** that we always use the same random number sequence. This makes the + ** tests repeatable. + */ + memset(zBuf, 0, nBuf); +#if !defined(SQLITE_TEST) + { + int pid, fd; + fd = robust_open("/dev/urandom", O_RDONLY, 0); + if( fd<0 ){ + time_t t; + time(&t); + memcpy(zBuf, &t, sizeof(t)); + pid = getpid(); + memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid)); + assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(pid); + }else{ + do{ nBuf = osRead(fd, zBuf, nBuf); }while( nBuf<0 && errno==EINTR ); + robust_close(0, fd, __LINE__); } } - /*NOTREACHED*/ - return SQLITE_ERROR; +#endif + return nBuf; } + /* -** Within this division (the proxying locking implementation) the procedures -** above this point are all utilities. The lock-related methods of the -** proxy-locking sqlite3_io_method object follow. +** Sleep for a little while. Return the amount of time slept. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. */ +static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ +#if OS_VXWORKS + struct timespec sp; + sp.tv_sec = microseconds / 1000000; + sp.tv_nsec = (microseconds % 1000000) * 1000; + nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#elif defined(HAVE_USLEEP) && HAVE_USLEEP + usleep(microseconds); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#else + int seconds = (microseconds+999999)/1000000; + sleep(seconds); + UNUSED_PARAMETER(NotUsed); + return seconds*1000000; +#endif +} /* -** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. */ -static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { - unixFile *pFile = (unixFile*)id; - int rc = proxyTakeConch(pFile); - if( rc==SQLITE_OK ){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - if( pCtx->conchHeld>0 ){ - unixFile *proxy = pCtx->lockProxy; - return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); - }else{ /* conchHeld < 0 is lockless */ - pResOut=0; - } - } - return rc; -} +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif /* -** Lock the file with the lock specified by parameter eFileLock - one -** of the following: -** -** (1) SHARED_LOCK -** (2) RESERVED_LOCK -** (3) PENDING_LOCK -** (4) EXCLUSIVE_LOCK -** -** Sometimes when requesting one lock state, additional lock states -** are inserted in between. The locking might fail on one of the later -** transitions leaving the lock state different from what it started but -** still short of its goal. The following chart shows the allowed -** transitions and the inserted intermediate states: -** -** UNLOCKED -> SHARED -** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE -** RESERVED -> (PENDING) -> EXCLUSIVE -** PENDING -> EXCLUSIVE +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. ** -** This routine will only increase a lock. Use the sqlite3OsUnlock() -** routine to lower a locking level. +** On success, return 0. Return 1 if the time and date cannot be found. */ -static int proxyLock(sqlite3_file *id, int eFileLock) { - unixFile *pFile = (unixFile*)id; - int rc = proxyTakeConch(pFile); - if( rc==SQLITE_OK ){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - if( pCtx->conchHeld>0 ){ - unixFile *proxy = pCtx->lockProxy; - rc = proxy->pMethod->xLock((sqlite3_file*)proxy, eFileLock); - pFile->eFileLock = proxy->eFileLock; - }else{ - /* conchHeld < 0 is lockless */ - } +static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; +#if defined(NO_GETTOD) + time_t t; + time(&t); + *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; +#elif OS_VXWORKS + struct timespec sNow; + clock_gettime(CLOCK_REALTIME, &sNow); + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; +#else + struct timeval sNow; + gettimeofday(&sNow, 0); + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; +#endif + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; } - return rc; +#endif + UNUSED_PARAMETER(NotUsed); + return 0; } - /* -** Lower the locking level on file descriptor pFile to eFileLock. eFileLock -** must be either NO_LOCK or SHARED_LOCK. -** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. */ -static int proxyUnlock(sqlite3_file *id, int eFileLock) { - unixFile *pFile = (unixFile*)id; - int rc = proxyTakeConch(pFile); - if( rc==SQLITE_OK ){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - if( pCtx->conchHeld>0 ){ - unixFile *proxy = pCtx->lockProxy; - rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, eFileLock); - pFile->eFileLock = proxy->eFileLock; - }else{ - /* conchHeld < 0 is lockless */ - } - } - return rc; +static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ + sqlite3_int64 i; + UNUSED_PARAMETER(NotUsed); + unixCurrentTimeInt64(0, &i); + *prNow = i/86400000.0; + return 0; } /* -** Close a file that uses proxy locks. +** We added the xGetLastError() method with the intention of providing +** better low-level error messages when operating-system problems come up +** during SQLite operation. But so far, none of that has been implemented +** in the core. So this routine is never called. For now, it is merely +** a place-holder. */ -static int proxyClose(sqlite3_file *id) { - if( id ){ - unixFile *pFile = (unixFile*)id; - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - unixFile *lockProxy = pCtx->lockProxy; - unixFile *conchFile = pCtx->conchFile; - int rc = SQLITE_OK; - - if( lockProxy ){ - rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); - if( rc ) return rc; - rc = lockProxy->pMethod->xClose((sqlite3_file*)lockProxy); - if( rc ) return rc; - sqlite3_free(lockProxy); - pCtx->lockProxy = 0; - } - if( conchFile ){ - if( pCtx->conchHeld ){ - rc = proxyReleaseConch(pFile); - if( rc ) return rc; - } - rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); - if( rc ) return rc; - sqlite3_free(conchFile); - } - sqlite3DbFree(0, pCtx->lockProxyPath); - sqlite3_free(pCtx->conchFilePath); - sqlite3DbFree(0, pCtx->dbPath); - /* restore the original locking context and pMethod then close it */ - pFile->lockingContext = pCtx->oldLockingContext; - pFile->pMethod = pCtx->pOldMethod; - sqlite3_free(pCtx); - return pFile->pMethod->xClose(id); - } - return SQLITE_OK; +static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ + UNUSED_PARAMETER(NotUsed); + UNUSED_PARAMETER(NotUsed2); + UNUSED_PARAMETER(NotUsed3); + return 0; } - -#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ /* -** The proxy locking style is intended for use with AFP filesystems. -** And since AFP is only supported on MacOSX, the proxy locking is also -** restricted to MacOSX. -** -** -******************* End of the proxy lock implementation ********************** +************************ End of sqlite3_vfs methods *************************** ******************************************************************************/ -/* -** Initialize the operating system interface. +/****************************************************************************** +************************** Begin Proxy Locking ******************************** ** -** This routine registers all VFS implementations for unix-like operating -** systems. This routine, and the sqlite3_os_end() routine that follows, -** should be the only routines in this file that are visible from other -** files. +** Proxy locking is a "uber-locking-method" in this sense: It uses the +** other locking methods on secondary lock files. Proxy locking is a +** meta-layer over top of the primitive locking implemented above. For +** this reason, the division that implements of proxy locking is deferred +** until late in the file (here) after all of the other I/O methods have +** been defined - so that the primitive locking methods are available +** as services to help with the implementation of proxy locking. ** -** This routine is called once during SQLite initialization and by a -** single thread. The memory allocation and mutex subsystems have not -** necessarily been initialized when this routine is called, and so they -** should not be used. -*/ -SQLITE_API int sqlite3_os_init(void){ - /* - ** The following macro defines an initializer for an sqlite3_vfs object. - ** The name of the VFS is NAME. The pAppData is a pointer to a pointer - ** to the "finder" function. (pAppData is a pointer to a pointer because - ** silly C90 rules prohibit a void* from being cast to a function pointer - ** and so we have to go through the intermediate pointer to avoid problems - ** when compiling with -pedantic-errors on GCC.) - ** - ** The FINDER parameter to this macro is the name of the pointer to the - ** finder-function. The finder-function returns a pointer to the - ** sqlite_io_methods object that implements the desired locking - ** behaviors. See the division above that contains the IOMETHODS - ** macro for addition information on finder-functions. - ** - ** Most finders simply return a pointer to a fixed sqlite3_io_methods - ** object. But the "autolockIoFinder" available on MacOSX does a little - ** more than that; it looks at the filesystem type that hosts the - ** database file and tries to choose an locking method appropriate for - ** that filesystem time. - */ - #define UNIXVFS(VFSNAME, FINDER) { \ - 2, /* iVersion */ \ - sizeof(unixFile), /* szOsFile */ \ - MAX_PATHNAME, /* mxPathname */ \ - 0, /* pNext */ \ - VFSNAME, /* zName */ \ - (void*)&FINDER, /* pAppData */ \ - unixOpen, /* xOpen */ \ - unixDelete, /* xDelete */ \ - unixAccess, /* xAccess */ \ - unixFullPathname, /* xFullPathname */ \ - unixDlOpen, /* xDlOpen */ \ - unixDlError, /* xDlError */ \ - unixDlSym, /* xDlSym */ \ - unixDlClose, /* xDlClose */ \ - unixRandomness, /* xRandomness */ \ - unixSleep, /* xSleep */ \ - unixCurrentTime, /* xCurrentTime */ \ - unixGetLastError, /* xGetLastError */ \ - unixCurrentTimeInt64, /* xCurrentTimeInt64 */ \ - } - - /* - ** All default VFSes for unix are contained in the following array. - ** - ** Note that the sqlite3_vfs.pNext field of the VFS object is modified - ** by the SQLite core when the VFS is registered. So the following - ** array cannot be const. - */ - static sqlite3_vfs aVfs[] = { -#if SQLITE_ENABLE_LOCKING_STYLE && (OS_VXWORKS || defined(__APPLE__)) - UNIXVFS("unix", autolockIoFinder ), -#else - UNIXVFS("unix", posixIoFinder ), -#endif - UNIXVFS("unix-none", nolockIoFinder ), - UNIXVFS("unix-dotfile", dotlockIoFinder ), -#if OS_VXWORKS - UNIXVFS("unix-namedsem", semIoFinder ), -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - UNIXVFS("unix-posix", posixIoFinder ), -#if !OS_VXWORKS - UNIXVFS("unix-flock", flockIoFinder ), -#endif -#endif -#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - UNIXVFS("unix-afp", afpIoFinder ), - UNIXVFS("unix-nfs", nfsIoFinder ), - UNIXVFS("unix-proxy", proxyIoFinder ), -#endif - }; - unsigned int i; /* Loop counter */ - - /* Register all VFSes defined in the aVfs[] array */ - for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ - sqlite3_vfs_register(&aVfs[i], i==0); - } - return SQLITE_OK; -} - -/* -** Shutdown the operating system interface. +**** ** -** Some operating systems might need to do some cleanup in this routine, -** to release dynamically allocated objects. But not on unix. -** This routine is a no-op for unix. -*/ -SQLITE_API int sqlite3_os_end(void){ - return SQLITE_OK; -} - -#endif /* SQLITE_OS_UNIX */ - -/************** End of os_unix.c *********************************************/ -/************** Begin file os_win.c ******************************************/ -/* -** 2004 May 22 +** The default locking schemes in SQLite use byte-range locks on the +** database file to coordinate safe, concurrent access by multiple readers +** and writers [http://sqlite.org/lockingv3.html]. The five file locking +** states (UNLOCKED, PENDING, SHARED, RESERVED, EXCLUSIVE) are implemented +** as POSIX read & write locks over fixed set of locations (via fsctl), +** on AFP and SMB only exclusive byte-range locks are available via fsctl +** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. +** To simulate a F_RDLCK on the shared range, on AFP a randomly selected +** address in the shared range is taken for a SHARED lock, the entire +** shared range is taken for an EXCLUSIVE lock): ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** PENDING_BYTE 0x40000000 +** RESERVED_BYTE 0x40000001 +** SHARED_RANGE 0x40000002 -> 0x40000200 ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This works well on the local file system, but shows a nearly 100x +** slowdown in read performance on AFP because the AFP client disables +** the read cache when byte-range locks are present. Enabling the read +** cache exposes a cache coherency problem that is present on all OS X +** supported network file systems. NFS and AFP both observe the +** close-to-open semantics for ensuring cache coherency +** [http://nfs.sourceforge.net/#faq_a8], which does not effectively +** address the requirements for concurrent database access by multiple +** readers and writers +** [http://www.nabble.com/SQLite-on-NFS-cache-coherency-td15655701.html]. ** -****************************************************************************** +** To address the performance and cache coherency issues, proxy file locking +** changes the way database access is controlled by limiting access to a +** single host at a time and moving file locks off of the database file +** and onto a proxy file on the local file system. ** -** This file contains code that is specific to windows. -*/ -#if SQLITE_OS_WIN /* This file is used for windows only */ - - -/* -** A Note About Memory Allocation: ** -** This driver uses malloc()/free() directly rather than going through -** the SQLite-wrappers sqlite3_malloc()/sqlite3_free(). Those wrappers -** are designed for use on embedded systems where memory is scarce and -** malloc failures happen frequently. Win32 does not typically run on -** embedded systems, and when it does the developers normally have bigger -** problems to worry about than running out of memory. So there is not -** a compelling need to use the wrappers. +** Using proxy locks +** ----------------- ** -** But there is a good reason to not use the wrappers. If we use the -** wrappers then we will get simulated malloc() failures within this -** driver. And that causes all kinds of problems for our tests. We -** could enhance SQLite to deal with simulated malloc failures within -** the OS driver, but the code to deal with those failure would not -** be exercised on Linux (which does not need to malloc() in the driver) -** and so we would have difficulty writing coverage tests for that -** code. Better to leave the code out, we think. +** C APIs ** -** The point of this discussion is as follows: When creating a new -** OS layer for an embedded system, if you use this file as an example, -** avoid the use of malloc()/free(). Those routines work ok on windows -** desktops but not so well in embedded systems. -*/ - -#include - -#ifdef __CYGWIN__ -# include -#endif - -/* -** Macros used to determine whether or not to use threads. -*/ -#if defined(THREADSAFE) && THREADSAFE -# define SQLITE_W32_THREADS 1 -#endif - -/* -** Include code that is common to all os_*.c files -*/ -/************** Include os_common.h in the middle of os_win.c ****************/ -/************** Begin file os_common.h ***************************************/ -/* -** 2004 May 22 +** sqlite3_file_control(db, dbname, SQLITE_SET_LOCKPROXYFILE, +** | ":auto:"); +** sqlite3_file_control(db, dbname, SQLITE_GET_LOCKPROXYFILE, &); ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** SQL pragmas ** -****************************************************************************** +** PRAGMA [database.]lock_proxy_file= | :auto: +** PRAGMA [database.]lock_proxy_file ** -** This file contains macros and a little bit of code that is common to -** all of the platform-specific files (os_*.c) and is #included into those -** files. +** Specifying ":auto:" means that if there is a conch file with a matching +** host ID in it, the proxy path in the conch file will be used, otherwise +** a proxy path based on the user's temp dir +** (via confstr(_CS_DARWIN_USER_TEMP_DIR,...)) will be used and the +** actual proxy file name is generated from the name and path of the +** database file. For example: ** -** This file should be #included by the os_*.c files only. It is not a -** general purpose header file. +** For database path "/Users/me/foo.db" +** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") +** +** Once a lock proxy is configured for a database connection, it can not +** be removed, however it may be switched to a different proxy path via +** the above APIs (assuming the conch file is not being held by another +** connection or process). +** +** +** How proxy locking works +** ----------------------- +** +** Proxy file locking relies primarily on two new supporting files: +** +** * conch file to limit access to the database file to a single host +** at a time +** +** * proxy file to act as a proxy for the advisory locks normally +** taken on the database +** +** The conch file - to use a proxy file, sqlite must first "hold the conch" +** by taking an sqlite-style shared lock on the conch file, reading the +** contents and comparing the host's unique host ID (see below) and lock +** proxy path against the values stored in the conch. The conch file is +** stored in the same directory as the database file and the file name +** is patterned after the database file name as ".-conch". +** If the conch file does not exist, or it's contents do not match the +** host ID and/or proxy path, then the lock is escalated to an exclusive +** lock and the conch file contents is updated with the host ID and proxy +** path and the lock is downgraded to a shared lock again. If the conch +** is held by another process (with a shared lock), the exclusive lock +** will fail and SQLITE_BUSY is returned. +** +** The proxy file - a single-byte file used for all advisory file locks +** normally taken on the database file. This allows for safe sharing +** of the database file for multiple readers and writers on the same +** host (the conch ensures that they all use the same local lock file). +** +** Requesting the lock proxy does not immediately take the conch, it is +** only taken when the first request to lock database file is made. +** This matches the semantics of the traditional locking behavior, where +** opening a connection to a database file does not take a lock on it. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. +** +** The proxy file and the lock file are never deleted so they only need +** to be created the first time they are used. +** +** Configuration options +** --------------------- +** +** SQLITE_PREFER_PROXY_LOCKING +** +** Database files accessed on non-local file systems are +** automatically configured for proxy locking, lock files are +** named automatically using the same logic as +** PRAGMA lock_proxy_file=":auto:" +** +** SQLITE_PROXY_DEBUG +** +** Enables the logging of error messages during host id file +** retrieval and creation +** +** LOCKPROXYDIR +** +** Overrides the default directory used for lock proxy files that +** are named automatically via the ":auto:" setting +** +** SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +** +** Permissions to use when creating a directory for storing the +** lock proxy files, only used when LOCKPROXYDIR is not set. +** +** +** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, +** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will +** force proxy locking to be used for every database file opened, and 0 +** will force automatic proxy locking to be disabled for all database +** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or +** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). */ -#ifndef _OS_COMMON_H_ -#define _OS_COMMON_H_ /* -** At least two bugs have slipped in because we changed the MEMORY_DEBUG -** macro to SQLITE_DEBUG and some older makefiles have not yet made the -** switch. The following code should catch this problem at compile-time. +** Proxy locking is only available on MacOSX */ -#ifdef MEMORY_DEBUG -# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." -#endif - -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3OSTrace = 0; -#define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X -#else -#define OSTRACE(X) -#endif +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE /* -** Macros for performance tracing. Normally turned off. Only works -** on i486 hardware. +** The proxyLockingContext has the path and file structures for the remote +** and local proxy files in it */ -#ifdef SQLITE_PERFORMANCE_TRACE +typedef struct proxyLockingContext proxyLockingContext; +struct proxyLockingContext { + unixFile *conchFile; /* Open conch file */ + char *conchFilePath; /* Name of the conch file */ + unixFile *lockProxy; /* Open proxy lock file */ + char *lockProxyPath; /* Name of the proxy lock file */ + char *dbPath; /* Name of the open file */ + int conchHeld; /* 1 if the conch is held, -1 if lockless */ + void *oldLockingContext; /* Original lockingcontext to restore on close */ + sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ +}; /* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. +** The proxy lock file path for the database at dbPath is written into lPath, +** which must point to valid, writable memory large enough for a maxLen length +** file path. */ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) +static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ + int len; + int dbLen; + int i; - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; +#ifdef LOCKPROXYDIR + len = strlcpy(lPath, LOCKPROXYDIR, maxLen); +#else +# ifdef _CS_DARWIN_USER_TEMP_DIR + { + if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ + OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", + lPath, errno, getpid())); + return SQLITE_IOERR_LOCK; + } + len = strlcat(lPath, "sqliteplocks", maxLen); } +# else + len = strlcpy(lPath, "/tmp/", maxLen); +# endif +#endif - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } + if( lPath[len-1]!='/' ){ + len = strlcat(lPath, "/", maxLen); } - - #endif - -#elif (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; + + /* transform the db path to a unique cache name */ + dbLen = (int)strlen(dbPath); + for( i=0; i 0) ){ + /* only mkdir if leaf dir != "." or "/" or ".." */ + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ + buf[i]='\0'; + if( mkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ + int err=errno; + if( err!=EEXIST ) { + OSTRACE(("CREATELOCKPATH FAILED creating %s, " + "'%s' proxy lock path=%s pid=%d\n", + buf, strerror(err), lockPath, getpid())); + return err; + } + } + } + start=i+1; + } + buf[i] = lockPath[i]; } + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid())); + return 0; +} -#else - - #error Need implementation of sqlite3Hwtime() for your platform. +/* +** Create a new VFS file descriptor (stored in memory obtained from +** sqlite3_malloc) and open the file named "path" in the file descriptor. +** +** The caller is responsible not only for closing the file descriptor +** but also for freeing the memory associated with the file descriptor. +*/ +static int proxyCreateUnixFile( + const char *path, /* path for the new unixFile */ + unixFile **ppFile, /* unixFile created and returned by ref */ + int islockfile /* if non zero missing dirs will be created */ +) { + int fd = -1; + int dirfd = -1; + unixFile *pNew; + int rc = SQLITE_OK; + int openFlags = O_RDWR | O_CREAT; + sqlite3_vfs dummyVfs; + int terrno = 0; + UnixUnusedFd *pUnused = NULL; - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. + /* 1. first try to open/create the file + ** 2. if that fails, and this is a lock file (not-conch), try creating + ** the parent directories and then try again. + ** 3. if that fails, try to open the file read-only + ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + pUnused = findReusableFd(path, openFlags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM; + } + } + if( fd<0 ){ + fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + terrno = errno; + if( fd<0 && errno==ENOENT && islockfile ){ + if( proxyCreateLockPath(path) == SQLITE_OK ){ + fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + } + } + } + if( fd<0 ){ + openFlags = O_RDONLY; + fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + terrno = errno; + } + if( fd<0 ){ + if( islockfile ){ + return SQLITE_BUSY; + } + switch (terrno) { + case EACCES: + return SQLITE_PERM; + case EIO: + return SQLITE_IOERR_LOCK; /* even though it is the conch */ + default: + return SQLITE_CANTOPEN_BKPT; + } + } + + pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); + if( pNew==NULL ){ + rc = SQLITE_NOMEM; + goto end_create_proxy; + } + memset(pNew, 0, sizeof(unixFile)); + pNew->openFlags = openFlags; + memset(&dummyVfs, 0, sizeof(dummyVfs)); + dummyVfs.pAppData = (void*)&autolockIoFinder; + dummyVfs.zName = "dummy"; + pUnused->fd = fd; + pUnused->flags = openFlags; + pNew->pUnused = pUnused; + + rc = fillInUnixFile(&dummyVfs, fd, dirfd, (sqlite3_file*)pNew, path, 0, 0, 0); + if( rc==SQLITE_OK ){ + *ppFile = pNew; + return SQLITE_OK; + } +end_create_proxy: + robust_close(pNew, fd, __LINE__); + sqlite3_free(pNew); + sqlite3_free(pUnused); + return rc; +} +#ifdef SQLITE_TEST +/* simulate multiple hosts by creating unique hostid file paths */ +SQLITE_API int sqlite3_hostid_num = 0; #endif -#endif /* !defined(_HWTIME_H_) */ +#define PROXY_HOSTIDLEN 16 /* conch file host id length */ -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ +/* Not always defined in the headers as it ought to be */ +extern int gethostuuid(uuid_t id, const struct timespec *wait); -static sqlite_uint64 g_start; -static sqlite_uint64 g_elapsed; -#define TIMER_START g_start=sqlite3Hwtime() -#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start -#define TIMER_ELAPSED g_elapsed -#else -#define TIMER_START -#define TIMER_END -#define TIMER_ELAPSED ((sqlite_uint64)0) +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +** bytes of writable memory. +*/ +static int proxyGetHostID(unsigned char *pHostID, int *pError){ + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memset(pHostID, 0, PROXY_HOSTIDLEN); +#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ + && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 + { + static const struct timespec timeout = {1, 0}; /* 1 sec timeout */ + if( gethostuuid(pHostID, &timeout) ){ + int err = errno; + if( pError ){ + *pError = err; + } + return SQLITE_IOERR; + } + } +#endif +#ifdef SQLITE_TEST + /* simulate multiple hosts by creating unique hostid file paths */ + if( sqlite3_hostid_num != 0){ + pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); + } #endif + + return SQLITE_OK; +} -/* -** If we compile with the SQLITE_TEST macro set, then the following block -** of code will give us the ability to simulate a disk I/O error. This -** is used for testing the I/O recovery logic. +/* The conch file contains the header, host id and lock file path + */ +#define PROXY_CONCHVERSION 2 /* 1-byte header, 16-byte host id, path */ +#define PROXY_HEADERLEN 1 /* conch file header length */ +#define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) +#define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) + +/* +** Takes an open conch file, copies the contents to a new path and then moves +** it back. The newly created file's file descriptor is assigned to the +** conch file structure and finally the original conch file descriptor is +** closed. Returns zero if successful. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ -SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ -SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ -SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ -SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ -SQLITE_API int sqlite3_diskfull_pending = 0; -SQLITE_API int sqlite3_diskfull = 0; -#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) -#define SimulateIOError(CODE) \ - if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ - || sqlite3_io_error_pending-- == 1 ) \ - { local_ioerr(); CODE; } -static void local_ioerr(){ - IOTRACE(("IOERR\n")); - sqlite3_io_error_hit++; - if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; -} -#define SimulateDiskfullError(CODE) \ - if( sqlite3_diskfull_pending ){ \ - if( sqlite3_diskfull_pending == 1 ){ \ - local_ioerr(); \ - sqlite3_diskfull = 1; \ - sqlite3_io_error_hit = 1; \ - CODE; \ - }else{ \ - sqlite3_diskfull_pending--; \ - } \ - } -#else -#define SimulateIOErrorBenign(X) -#define SimulateIOError(A) -#define SimulateDiskfullError(A) -#endif - -/* -** When testing, keep a count of the number of open files. -*/ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_open_file_count = 0; -#define OpenCounter(X) sqlite3_open_file_count+=(X) -#else -#define OpenCounter(X) -#endif +static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + char tPath[MAXPATHLEN]; + char buf[PROXY_MAXCONCHLEN]; + char *cPath = pCtx->conchFilePath; + size_t readLen = 0; + size_t pathLen = 0; + char errmsg[64] = ""; + int fd = -1; + int rc = -1; + UNUSED_PARAMETER(myHostID); -#endif /* !defined(_OS_COMMON_H_) */ + /* create a new path by replace the trailing '-conch' with '-break' */ + pathLen = strlcpy(tPath, cPath, MAXPATHLEN); + if( pathLen>MAXPATHLEN || pathLen<6 || + (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ + sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); + goto end_breaklock; + } + /* read the conch content */ + readLen = osPread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); + if( readLenh, __LINE__); + conchFile->h = fd; + conchFile->openFlags = O_RDWR | O_CREAT; -/************** End of os_common.h *******************************************/ -/************** Continuing where we left off in os_win.c *********************/ +end_breaklock: + if( rc ){ + if( fd>=0 ){ + unlink(tPath); + robust_close(pFile, fd, __LINE__); + } + fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); + } + return rc; +} -/* -** Some microsoft compilers lack this definition. +/* Take the requested lock on the conch file and break a stale lock if the +** host id matches. */ -#ifndef INVALID_FILE_ATTRIBUTES -# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) -#endif +static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + int nTries = 0; + struct timespec conchModTime; + + do { + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + nTries ++; + if( rc==SQLITE_BUSY ){ + /* If the lock failed (busy): + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait + * 10 sec and try again + * 3rd try: break the lock unless the mod time has changed. + */ + struct stat buf; + if( osFstat(conchFile->h, &buf) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_LOCK; + } + + if( nTries==1 ){ + conchModTime = buf.st_mtimespec; + usleep(500000); /* wait 0.5 sec and try the lock again*/ + continue; + } -/* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. + assert( nTries>1 ); + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ + return SQLITE_BUSY; + } + + if( nTries==2 ){ + char tBuf[PROXY_MAXCONCHLEN]; + int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + if( len<0 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_LOCK; + } + if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ + /* don't break the lock if the host id doesn't match */ + if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ + return SQLITE_BUSY; + } + }else{ + /* don't break the lock on short read or a version mismatch */ + return SQLITE_BUSY; + } + usleep(10000000); /* wait 10 sec and try the lock again */ + continue; + } + + assert( nTries==3 ); + if( 0==proxyBreakConchLock(pFile, myHostID) ){ + rc = SQLITE_OK; + if( lockType==EXCLUSIVE_LOCK ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + } + if( !rc ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + } + } + } + } while( rc==SQLITE_BUSY && nTries<3 ); + + return rc; +} + +/* Takes the conch by taking a shared lock and read the contents conch, if +** lockPath is non-NULL, the host ID and lock file path must match. A NULL +** lockPath means that the lockPath in the conch file will be used if the +** host IDs match, or a new lock path will be generated automatically +** and written to the conch file. */ -#if SQLITE_OS_WINCE -# define AreFileApisANSI() 1 -# define FormatMessageW(a,b,c,d,e,f,g) 0 -#endif +static int proxyTakeConch(unixFile *pFile){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + + if( pCtx->conchHeld!=0 ){ + return SQLITE_OK; + }else{ + unixFile *conchFile = pCtx->conchFile; + uuid_t myHostID; + int pError = 0; + char readBuf[PROXY_MAXCONCHLEN]; + char lockPath[MAXPATHLEN]; + char *tempLockPath = NULL; + int rc = SQLITE_OK; + int createConch = 0; + int hostIdMatch = 0; + int readLen = 0; + int tryOldLockPath = 0; + int forceNewLockPath = 0; + + OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid())); -/* Forward references */ -typedef struct winShm winShm; /* A connection to shared-memory */ -typedef struct winShmNode winShmNode; /* A region of shared-memory */ + rc = proxyGetHostID(myHostID, &pError); + if( (rc&0xff)==SQLITE_IOERR ){ + pFile->lastErrno = pError; + goto end_takeconch; + } + rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + goto end_takeconch; + } + /* read the existing conch file */ + readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); + if( readLen<0 ){ + /* I/O error: lastErrno set by seekAndRead */ + pFile->lastErrno = conchFile->lastErrno; + rc = SQLITE_IOERR_READ; + goto end_takeconch; + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + readBuf[0]!=(char)PROXY_CONCHVERSION ){ + /* a short read or version format mismatch means we need to create a new + ** conch file. + */ + createConch = 1; + } + /* if the host id matches and the lock path already exists in the conch + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path + */ + do { /* in case we need to try again for an :auto: named lock file */ -/* -** WinCE lacks native support for file locking so we have to fake it -** with some code of our own. -*/ -#if SQLITE_OS_WINCE -typedef struct winceLock { - int nReaders; /* Number of reader locks obtained */ - BOOL bPending; /* Indicates a pending lock has been obtained */ - BOOL bReserved; /* Indicates a reserved lock has been obtained */ - BOOL bExclusive; /* Indicates an exclusive lock has been obtained */ -} winceLock; + if( !createConch && !forceNewLockPath ){ + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + PROXY_HOSTIDLEN); + /* if the conch has data compare the contents */ + if( !pCtx->lockProxyPath ){ + /* for auto-named local lock file, just check the host ID and we'll + ** use the local lock file path that's already in there + */ + if( hostIdMatch ){ + size_t pathLen = (readLen - PROXY_PATHINDEX); + + if( pathLen>=MAXPATHLEN ){ + pathLen=MAXPATHLEN-1; + } + memcpy(lockPath, &readBuf[PROXY_PATHINDEX], pathLen); + lockPath[pathLen] = 0; + tempLockPath = lockPath; + tryOldLockPath = 1; + /* create a copy of the lock path if the conch is taken */ + goto end_takeconch; + } + }else if( hostIdMatch + && !strncmp(pCtx->lockProxyPath, &readBuf[PROXY_PATHINDEX], + readLen-PROXY_PATHINDEX) + ){ + /* conch host and lock path match */ + goto end_takeconch; + } + } + + /* if the conch isn't writable and doesn't match, we can't take it */ + if( (conchFile->openFlags&O_RDWR) == 0 ){ + rc = SQLITE_BUSY; + goto end_takeconch; + } + + /* either the conch didn't match or we need to create a new one */ + if( !pCtx->lockProxyPath ){ + proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); + tempLockPath = lockPath; + /* create a copy of the lock path _only_ if the conch is taken */ + } + + /* update conch with host and path (this will fail if other process + ** has a shared lock already), if the host id matches, use the big + ** stick. + */ + futimes(conchFile->h, NULL); + if( hostIdMatch && !createConch ){ + if( conchFile->pInode && conchFile->pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + } else { + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); + } + }else{ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); + } + if( rc==SQLITE_OK ){ + char writeBuffer[PROXY_MAXCONCHLEN]; + int writeSize = 0; + + writeBuffer[0] = (char)PROXY_CONCHVERSION; + memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); + if( pCtx->lockProxyPath!=NULL ){ + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); + }else{ + strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); + } + writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); + robust_ftruncate(conchFile->h, writeSize); + rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); + fsync(conchFile->h); + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database + */ + if( rc==SQLITE_OK && createConch ){ + struct stat buf; + int err = osFstat(pFile->h, &buf); + if( err==0 ){ + mode_t cmode = buf.st_mode&(S_IRUSR|S_IWUSR | S_IRGRP|S_IWGRP | + S_IROTH|S_IWOTH); + /* try to match the database file R/W permissions, ignore failure */ +#ifndef SQLITE_PROXY_DEBUG + osFchmod(conchFile->h, cmode); +#else + do{ + rc = osFchmod(conchFile->h, cmode); + }while( rc==(-1) && errno==EINTR ); + if( rc!=0 ){ + int code = errno; + fprintf(stderr, "fchmod %o FAILED with %d %s\n", + cmode, code, strerror(code)); + } else { + fprintf(stderr, "fchmod %o SUCCEDED\n",cmode); + } + }else{ + int code = errno; + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + err, code, strerror(code)); #endif + } + } + } + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); + + end_takeconch: + OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); + if( rc==SQLITE_OK && pFile->openFlags ){ + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + } + pFile->h = -1; + int fd = robust_open(pCtx->dbPath, pFile->openFlags, + SQLITE_DEFAULT_FILE_PERMISSIONS); + OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); + if( fd>=0 ){ + pFile->h = fd; + }else{ + rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called + during locking */ + } + } + if( rc==SQLITE_OK && !pCtx->lockProxy ){ + char *path = tempLockPath ? tempLockPath : pCtx->lockProxyPath; + rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ + /* we couldn't create the proxy lock file with the old lock file path + ** so try again via auto-naming + */ + forceNewLockPath = 1; + tryOldLockPath = 0; + continue; /* go back to the do {} while start point, try again */ + } + } + if( rc==SQLITE_OK ){ + /* Need to make a copy of path if we extracted the value + ** from the conch file or the path was allocated on the stack + */ + if( tempLockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); + if( !pCtx->lockProxyPath ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + pCtx->conchHeld = 1; + + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ + afpLockingContext *afpCtx; + afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; + afpCtx->dbPath = pCtx->lockProxyPath; + } + } else { + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + OSTRACE(("TAKECONCH %d %s\n", conchFile->h, + rc==SQLITE_OK?"ok":"failed")); + return rc; + } while (1); /* in case we need to retry the :auto: lock file - + ** we should never get here except via the 'continue' call. */ + } +} /* -** The winFile structure is a subclass of sqlite3_file* specific to the win32 -** portability layer. +** If pFile holds a lock on a conch file, then release that lock. */ -typedef struct winFile winFile; -struct winFile { - const sqlite3_io_methods *pMethod; /*** Must be first ***/ - sqlite3_vfs *pVfs; /* The VFS used to open this file */ - HANDLE h; /* Handle for accessing the file */ - unsigned char locktype; /* Type of lock currently held on this file */ - short sharedLockByte; /* Randomly chosen byte used as a shared lock */ - DWORD lastErrno; /* The Windows errno from the last I/O error */ - DWORD sectorSize; /* Sector size of the device file is on */ - winShm *pShm; /* Instance of shared memory on this file */ - const char *zPath; /* Full pathname of this file */ - int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ -#if SQLITE_OS_WINCE - WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ - HANDLE hMutex; /* Mutex used to control access to shared lock */ - HANDLE hShared; /* Shared memory segment used for locking */ - winceLock local; /* Locks obtained by this instance of winFile */ - winceLock *shared; /* Global shared lock memory for the file */ -#endif -}; +static int proxyReleaseConch(unixFile *pFile){ + int rc = SQLITE_OK; /* Subroutine return code */ + proxyLockingContext *pCtx; /* The locking context for the proxy lock */ + unixFile *conchFile; /* Name of the conch file */ -/* -** Forward prototypes. -*/ -static int getSectorSize( - sqlite3_vfs *pVfs, - const char *zRelative /* UTF-8 file name */ -); + pCtx = (proxyLockingContext *)pFile->lockingContext; + conchFile = pCtx->conchFile; + OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + getpid())); + if( pCtx->conchHeld>0 ){ + rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + pCtx->conchHeld = 0; + OSTRACE(("RELEASECONCH %d %s\n", conchFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} /* -** The following variable is (normally) set once and never changes -** thereafter. It records whether the operating system is Win95 -** or WinNT. +** Given the name of a database file, compute the name of its conch file. +** Store the conch filename in memory obtained from sqlite3_malloc(). +** Make *pConchPath point to the new name. Return SQLITE_OK on success +** or SQLITE_NOMEM if unable to obtain memory. ** -** 0: Operating system unknown. -** 1: Operating system is Win95. -** 2: Operating system is WinNT. +** The caller is responsible for ensuring that the allocated memory +** space is eventually freed. ** -** In order to facilitate testing on a WinNT system, the test fixture -** can manually set this value to 1 to emulate Win98 behavior. +** *pConchPath is set to NULL if a memory allocation error occurs. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_os_type = 0; -#else -static int sqlite3_os_type = 0; -#endif +static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ + int i; /* Loop counter */ + int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ + char *conchPath; /* buffer in which to construct conch name */ -/* -** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, -** or WinCE. Return false (zero) for Win95, Win98, or WinME. -** -** Here is an interesting observation: Win95, Win98, and WinME lack -** the LockFileEx() API. But we can still statically link against that -** API as long as we don't call it when running Win95/98/ME. A call to -** this routine is used to determine if the host is Win95/98/ME or -** WinNT/2K/XP so that we will know whether or not we can safely call -** the LockFileEx() API. -*/ -#if SQLITE_OS_WINCE -# define isNT() (1) -#else - static int isNT(void){ - if( sqlite3_os_type==0 ){ - OSVERSIONINFO sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - GetVersionEx(&sInfo); - sqlite3_os_type = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; + /* Allocate space for the conch filename and initialize the name to + ** the name of the original database file. */ + *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); + if( conchPath==0 ){ + return SQLITE_NOMEM; + } + memcpy(conchPath, dbPath, len+1); + + /* now insert a "." before the last / character */ + for( i=(len-1); i>=0; i-- ){ + if( conchPath[i]=='/' ){ + i++; + break; } - return sqlite3_os_type==2; } -#endif /* SQLITE_OS_WINCE */ + conchPath[i]='.'; + while ( ilockingContext; + char *oldPath = pCtx->lockProxyPath; + int rc = SQLITE_OK; - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); - zFilename = malloc( nByte ); - if( zFilename==0 ){ - return 0; - } - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, - 0, 0); - if( nByte == 0 ){ - free(zFilename); - zFilename = 0; + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + + /* nothing to do if the path is NULL, :auto: or matches the existing path */ + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || + (oldPath && !strncmp(oldPath, path, MAXPATHLEN)) ){ + return SQLITE_OK; + }else{ + unixFile *lockProxy = pCtx->lockProxy; + pCtx->lockProxy=NULL; + pCtx->conchHeld = 0; + if( lockProxy!=NULL ){ + rc=lockProxy->pMethod->xClose((sqlite3_file *)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + } + sqlite3_free(oldPath); + pCtx->lockProxyPath = sqlite3DbStrDup(0, path); } - return zFilename; + + return rc; } /* -** Convert an ansi string to microsoft unicode, based on the -** current codepage settings for file apis. -** -** Space to hold the returned string is obtained -** from malloc. +** pFile is a file that has been opened by a prior xOpen call. dbPath +** is a string buffer at least MAXPATHLEN+1 characters in size. +** +** This routine find the filename associated with pFile and writes it +** int dbPath. */ -static WCHAR *mbcsToUnicode(const char *zFilename){ - int nByte; - WCHAR *zMbcsFilename; - int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; - - nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, NULL,0)*sizeof(WCHAR); - zMbcsFilename = malloc( nByte*sizeof(zMbcsFilename[0]) ); - if( zMbcsFilename==0 ){ - return 0; - } - nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, nByte); - if( nByte==0 ){ - free(zMbcsFilename); - zMbcsFilename = 0; +static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ +#if defined(__APPLE__) + if( pFile->pMethod == &afpIoMethods ){ + /* afp style keeps a reference to the db path in the filePath field + ** of the struct */ + assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, MAXPATHLEN); + } else +#endif + if( pFile->pMethod == &dotlockIoMethods ){ + /* dot lock style uses the locking context to store the dot lock + ** file path */ + int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); + memcpy(dbPath, (char *)pFile->lockingContext, len + 1); + }else{ + /* all other styles use the locking context to store the db file path */ + assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); } - return zMbcsFilename; + return SQLITE_OK; } /* -** Convert microsoft unicode to multibyte character string, based on the -** user's Ansi codepage. -** -** Space to hold the returned string is obtained from -** malloc(). +** Takes an already filled in unix file and alters it so all file locking +** will be performed on the local proxy lock file. The following fields +** are preserved in the locking context so that they can be restored and +** the unix structure properly cleaned up at close time: +** ->lockingContext +** ->pMethod */ -static char *unicodeToMbcs(const WCHAR *zWideFilename){ - int nByte; - char *zFilename; - int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; - - nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); - zFilename = malloc( nByte ); - if( zFilename==0 ){ - return 0; +static int proxyTransformUnixFile(unixFile *pFile, const char *path) { + proxyLockingContext *pCtx; + char dbPath[MAXPATHLEN+1]; /* Name of the database file */ + char *lockPath=NULL; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; } - nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, nByte, - 0, 0); - if( nByte == 0 ){ - free(zFilename); - zFilename = 0; + proxyGetDbPathForUnixFile(pFile, dbPath); + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ + lockPath=NULL; + }else{ + lockPath=(char *)path; } - return zFilename; -} + + OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, + (lockPath ? lockPath : ":auto:"), getpid())); -/* -** Convert multibyte character string to UTF-8. Space to hold the -** returned string is obtained from malloc(). -*/ -SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ - char *zFilenameUtf8; - WCHAR *zTmpWide; + pCtx = sqlite3_malloc( sizeof(*pCtx) ); + if( pCtx==0 ){ + return SQLITE_NOMEM; + } + memset(pCtx, 0, sizeof(*pCtx)); - zTmpWide = mbcsToUnicode(zFilename); - if( zTmpWide==0 ){ - return 0; + rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); + if( rc==SQLITE_OK ){ + rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile, 0); + if( rc==SQLITE_CANTOPEN && ((pFile->openFlags&O_RDWR) == 0) ){ + /* if (a) the open flags are not O_RDWR, (b) the conch isn't there, and + ** (c) the file system is read-only, then enable no-locking access. + ** Ugh, since O_RDONLY==0x0000 we test for !O_RDWR since unixOpen asserts + ** that openFlags will have only one of O_RDONLY or O_RDWR. + */ + struct statfs fsInfo; + struct stat conchInfo; + int goLockless = 0; + + if( osStat(pCtx->conchFilePath, &conchInfo) == -1 ) { + int err = errno; + if( (err==ENOENT) && (statfs(dbPath, &fsInfo) != -1) ){ + goLockless = (fsInfo.f_flags&MNT_RDONLY) == MNT_RDONLY; + } + } + if( goLockless ){ + pCtx->conchHeld = -1; /* read only FS/ lockless */ + rc = SQLITE_OK; + } + } + } + if( rc==SQLITE_OK && lockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); } - zFilenameUtf8 = unicodeToUtf8(zTmpWide); - free(zTmpWide); - return zFilenameUtf8; + + if( rc==SQLITE_OK ){ + pCtx->dbPath = sqlite3DbStrDup(0, dbPath); + if( pCtx->dbPath==NULL ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + /* all memory is allocated, proxys are created and assigned, + ** switch the locking context and pMethod then return. + */ + pCtx->oldLockingContext = pFile->lockingContext; + pFile->lockingContext = pCtx; + pCtx->pOldMethod = pFile->pMethod; + pFile->pMethod = &proxyIoMethods; + }else{ + if( pCtx->conchFile ){ + pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); + sqlite3_free(pCtx->conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx); + } + OSTRACE(("TRANSPROXY %d %s\n", pFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; } + /* -** Convert UTF-8 to multibyte character string. Space to hold the -** returned string is obtained from malloc(). +** This routine handles sqlite3_file_control() calls that are specific +** to proxy locking. */ -static char *utf8ToMbcs(const char *zFilename){ - char *zFilenameMbcs; - WCHAR *zTmpWide; - - zTmpWide = utf8ToUnicode(zFilename); - if( zTmpWide==0 ){ - return 0; +static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_GET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + if( pFile->pMethod == &proxyIoMethods ){ + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; + proxyTakeConch(pFile); + if( pCtx->lockProxyPath ){ + *(const char **)pArg = pCtx->lockProxyPath; + }else{ + *(const char **)pArg = ":auto: (not held)"; + } + } else { + *(const char **)pArg = NULL; + } + return SQLITE_OK; + } + case SQLITE_SET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + int rc = SQLITE_OK; + int isProxyStyle = (pFile->pMethod == &proxyIoMethods); + if( pArg==NULL || (const char *)pArg==0 ){ + if( isProxyStyle ){ + /* turn off proxy locking - not supported */ + rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; + }else{ + /* turn off proxy locking - already off - NOOP */ + rc = SQLITE_OK; + } + }else{ + const char *proxyPath = (const char *)pArg; + if( isProxyStyle ){ + proxyLockingContext *pCtx = + (proxyLockingContext*)pFile->lockingContext; + if( !strcmp(pArg, ":auto:") + || (pCtx->lockProxyPath && + !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) + ){ + rc = SQLITE_OK; + }else{ + rc = switchLockProxyPath(pFile, proxyPath); + } + }else{ + /* turn on proxy file locking */ + rc = proxyTransformUnixFile(pFile, proxyPath); + } + } + return rc; + } + default: { + assert( 0 ); /* The call assures that only valid opcodes are sent */ + } } - zFilenameMbcs = unicodeToMbcs(zTmpWide); - free(zTmpWide); - return zFilenameMbcs; + /*NOTREACHED*/ + return SQLITE_ERROR; } -#if SQLITE_OS_WINCE -/************************************************************************* -** This section contains code for WinCE only. -*/ /* -** WindowsCE does not have a localtime() function. So create a -** substitute. +** Within this division (the proxying locking implementation) the procedures +** above this point are all utilities. The lock-related methods of the +** proxy-locking sqlite3_io_method object follow. */ -struct tm *__cdecl localtime(const time_t *t) -{ - static struct tm y; - FILETIME uTm, lTm; - SYSTEMTIME pTm; - sqlite3_int64 t64; - t64 = *t; - t64 = (t64 + 11644473600)*10000000; - uTm.dwLowDateTime = (DWORD)(t64 & 0xFFFFFFFF); - uTm.dwHighDateTime= (DWORD)(t64 >> 32); - FileTimeToLocalFileTime(&uTm,&lTm); - FileTimeToSystemTime(&lTm,&pTm); - y.tm_year = pTm.wYear - 1900; - y.tm_mon = pTm.wMonth - 1; - y.tm_wday = pTm.wDayOfWeek; - y.tm_mday = pTm.wDay; - y.tm_hour = pTm.wHour; - y.tm_min = pTm.wMinute; - y.tm_sec = pTm.wSecond; - return &y; -} - -/* This will never be called, but defined to make the code compile */ -#define GetTempPathA(a,b) - -#define LockFile(a,b,c,d,e) winceLockFile(&a, b, c, d, e) -#define UnlockFile(a,b,c,d,e) winceUnlockFile(&a, b, c, d, e) -#define LockFileEx(a,b,c,d,e,f) winceLockFileEx(&a, b, c, d, e, f) -#define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] /* -** Acquire a lock on the handle h +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static void winceMutexAcquire(HANDLE h){ - DWORD dwErr; - do { - dwErr = WaitForSingleObject(h, INFINITE); - } while (dwErr != WAIT_OBJECT_0 && dwErr != WAIT_ABANDONED); +static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + }else{ /* conchHeld < 0 is lockless */ + pResOut=0; + } + } + return rc; } -/* -** Release a lock acquired by winceMutexAcquire() -*/ -#define winceMutexRelease(h) ReleaseMutex(h) /* -** Create the mutex and shared memory used for locking in the file -** descriptor pFile +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. */ -static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ - WCHAR *zTok; - WCHAR *zName = utf8ToUnicode(zFilename); - BOOL bInit = TRUE; - - /* Initialize the local lockdata */ - ZeroMemory(&pFile->local, sizeof(pFile->local)); - - /* Replace the backslashes from the filename and lowercase it - ** to derive a mutex name. */ - zTok = CharLowerW(zName); - for (;*zTok;zTok++){ - if (*zTok == '\\') *zTok = '_'; - } - - /* Create/open the named mutex */ - pFile->hMutex = CreateMutexW(NULL, FALSE, zName); - if (!pFile->hMutex){ - pFile->lastErrno = GetLastError(); - free(zName); - return FALSE; - } - - /* Acquire the mutex before continuing */ - winceMutexAcquire(pFile->hMutex); - - /* Since the names of named mutexes, semaphores, file mappings etc are - ** case-sensitive, take advantage of that by uppercasing the mutex name - ** and using that as the shared filemapping name. - */ - CharUpperW(zName); - pFile->hShared = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, - PAGE_READWRITE, 0, sizeof(winceLock), - zName); - - /* Set a flag that indicates we're the first to create the memory so it - ** must be zero-initialized */ - if (GetLastError() == ERROR_ALREADY_EXISTS){ - bInit = FALSE; - } - - free(zName); - - /* If we succeeded in making the shared memory handle, map it. */ - if (pFile->hShared){ - pFile->shared = (winceLock*)MapViewOfFile(pFile->hShared, - FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); - /* If mapping failed, close the shared memory handle and erase it */ - if (!pFile->shared){ - pFile->lastErrno = GetLastError(); - CloseHandle(pFile->hShared); - pFile->hShared = NULL; +static int proxyLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xLock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ } } - - /* If shared memory could not be created, then close the mutex and fail */ - if (pFile->hShared == NULL){ - winceMutexRelease(pFile->hMutex); - CloseHandle(pFile->hMutex); - pFile->hMutex = NULL; - return FALSE; - } - - /* Initialize the shared memory if we're supposed to */ - if (bInit) { - ZeroMemory(pFile->shared, sizeof(winceLock)); - } - - winceMutexRelease(pFile->hMutex); - return TRUE; + return rc; } + /* -** Destroy the part of winFile that deals with wince locks +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. */ -static void winceDestroyLock(winFile *pFile){ - if (pFile->hMutex){ - /* Acquire the mutex */ - winceMutexAcquire(pFile->hMutex); - - /* The following blocks should probably assert in debug mode, but they - are to cleanup in case any locks remained open */ - if (pFile->local.nReaders){ - pFile->shared->nReaders --; - } - if (pFile->local.bReserved){ - pFile->shared->bReserved = FALSE; - } - if (pFile->local.bPending){ - pFile->shared->bPending = FALSE; - } - if (pFile->local.bExclusive){ - pFile->shared->bExclusive = FALSE; +static int proxyUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ } - - /* De-reference and close our copy of the shared memory handle */ - UnmapViewOfFile(pFile->shared); - CloseHandle(pFile->hShared); - - /* Done with the mutex */ - winceMutexRelease(pFile->hMutex); - CloseHandle(pFile->hMutex); - pFile->hMutex = NULL; } + return rc; } -/* -** An implementation of the LockFile() API of windows for wince +/* +** Close a file that uses proxy locks. */ -static BOOL winceLockFile( - HANDLE *phFile, - DWORD dwFileOffsetLow, - DWORD dwFileOffsetHigh, - DWORD nNumberOfBytesToLockLow, - DWORD nNumberOfBytesToLockHigh -){ - winFile *pFile = HANDLE_TO_WINFILE(phFile); - BOOL bReturn = FALSE; - - UNUSED_PARAMETER(dwFileOffsetHigh); - UNUSED_PARAMETER(nNumberOfBytesToLockHigh); - - if (!pFile->hMutex) return TRUE; - winceMutexAcquire(pFile->hMutex); - - /* Wanting an exclusive lock? */ - if (dwFileOffsetLow == (DWORD)SHARED_FIRST - && nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ - if (pFile->shared->nReaders == 0 && pFile->shared->bExclusive == 0){ - pFile->shared->bExclusive = TRUE; - pFile->local.bExclusive = TRUE; - bReturn = TRUE; +static int proxyClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *lockProxy = pCtx->lockProxy; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + + if( lockProxy ){ + rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); + if( rc ) return rc; + rc = lockProxy->pMethod->xClose((sqlite3_file*)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + pCtx->lockProxy = 0; } - } - - /* Want a read-only lock? */ - else if (dwFileOffsetLow == (DWORD)SHARED_FIRST && - nNumberOfBytesToLockLow == 1){ - if (pFile->shared->bExclusive == 0){ - pFile->local.nReaders ++; - if (pFile->local.nReaders == 1){ - pFile->shared->nReaders ++; + if( conchFile ){ + if( pCtx->conchHeld ){ + rc = proxyReleaseConch(pFile); + if( rc ) return rc; } - bReturn = TRUE; + rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); + if( rc ) return rc; + sqlite3_free(conchFile); } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3DbFree(0, pCtx->dbPath); + /* restore the original locking context and pMethod then close it */ + pFile->lockingContext = pCtx->oldLockingContext; + pFile->pMethod = pCtx->pOldMethod; + sqlite3_free(pCtx); + return pFile->pMethod->xClose(id); } + return SQLITE_OK; +} - /* Want a pending lock? */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToLockLow == 1){ - /* If no pending lock has been acquired, then acquire it */ - if (pFile->shared->bPending == 0) { - pFile->shared->bPending = TRUE; - pFile->local.bPending = TRUE; - bReturn = TRUE; - } - } - /* Want a reserved lock? */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToLockLow == 1){ - if (pFile->shared->bReserved == 0) { - pFile->shared->bReserved = TRUE; - pFile->local.bReserved = TRUE; - bReturn = TRUE; - } - } - winceMutexRelease(pFile->hMutex); - return bReturn; -} +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The proxy locking style is intended for use with AFP filesystems. +** And since AFP is only supported on MacOSX, the proxy locking is also +** restricted to MacOSX. +** +** +******************* End of the proxy lock implementation ********************** +******************************************************************************/ /* -** An implementation of the UnlockFile API of windows for wince +** Initialize the operating system interface. +** +** This routine registers all VFS implementations for unix-like operating +** systems. This routine, and the sqlite3_os_end() routine that follows, +** should be the only routines in this file that are visible from other +** files. +** +** This routine is called once during SQLite initialization and by a +** single thread. The memory allocation and mutex subsystems have not +** necessarily been initialized when this routine is called, and so they +** should not be used. */ -static BOOL winceUnlockFile( - HANDLE *phFile, - DWORD dwFileOffsetLow, - DWORD dwFileOffsetHigh, - DWORD nNumberOfBytesToUnlockLow, - DWORD nNumberOfBytesToUnlockHigh -){ - winFile *pFile = HANDLE_TO_WINFILE(phFile); - BOOL bReturn = FALSE; - - UNUSED_PARAMETER(dwFileOffsetHigh); - UNUSED_PARAMETER(nNumberOfBytesToUnlockHigh); - - if (!pFile->hMutex) return TRUE; - winceMutexAcquire(pFile->hMutex); +SQLITE_API int sqlite3_os_init(void){ + /* + ** The following macro defines an initializer for an sqlite3_vfs object. + ** The name of the VFS is NAME. The pAppData is a pointer to a pointer + ** to the "finder" function. (pAppData is a pointer to a pointer because + ** silly C90 rules prohibit a void* from being cast to a function pointer + ** and so we have to go through the intermediate pointer to avoid problems + ** when compiling with -pedantic-errors on GCC.) + ** + ** The FINDER parameter to this macro is the name of the pointer to the + ** finder-function. The finder-function returns a pointer to the + ** sqlite_io_methods object that implements the desired locking + ** behaviors. See the division above that contains the IOMETHODS + ** macro for addition information on finder-functions. + ** + ** Most finders simply return a pointer to a fixed sqlite3_io_methods + ** object. But the "autolockIoFinder" available on MacOSX does a little + ** more than that; it looks at the filesystem type that hosts the + ** database file and tries to choose an locking method appropriate for + ** that filesystem time. + */ + #define UNIXVFS(VFSNAME, FINDER) { \ + 3, /* iVersion */ \ + sizeof(unixFile), /* szOsFile */ \ + MAX_PATHNAME, /* mxPathname */ \ + 0, /* pNext */ \ + VFSNAME, /* zName */ \ + (void*)&FINDER, /* pAppData */ \ + unixOpen, /* xOpen */ \ + unixDelete, /* xDelete */ \ + unixAccess, /* xAccess */ \ + unixFullPathname, /* xFullPathname */ \ + unixDlOpen, /* xDlOpen */ \ + unixDlError, /* xDlError */ \ + unixDlSym, /* xDlSym */ \ + unixDlClose, /* xDlClose */ \ + unixRandomness, /* xRandomness */ \ + unixSleep, /* xSleep */ \ + unixCurrentTime, /* xCurrentTime */ \ + unixGetLastError, /* xGetLastError */ \ + unixCurrentTimeInt64, /* xCurrentTimeInt64 */ \ + unixSetSystemCall, /* xSetSystemCall */ \ + unixGetSystemCall, /* xGetSystemCall */ \ + unixNextSystemCall, /* xNextSystemCall */ \ + } - /* Releasing a reader lock or an exclusive lock */ - if (dwFileOffsetLow == (DWORD)SHARED_FIRST){ - /* Did we have an exclusive lock? */ - if (pFile->local.bExclusive){ - assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE); - pFile->local.bExclusive = FALSE; - pFile->shared->bExclusive = FALSE; - bReturn = TRUE; - } + /* + ** All default VFSes for unix are contained in the following array. + ** + ** Note that the sqlite3_vfs.pNext field of the VFS object is modified + ** by the SQLite core when the VFS is registered. So the following + ** array cannot be const. + */ + static sqlite3_vfs aVfs[] = { +#if SQLITE_ENABLE_LOCKING_STYLE && (OS_VXWORKS || defined(__APPLE__)) + UNIXVFS("unix", autolockIoFinder ), +#else + UNIXVFS("unix", posixIoFinder ), +#endif + UNIXVFS("unix-none", nolockIoFinder ), + UNIXVFS("unix-dotfile", dotlockIoFinder ), + UNIXVFS("unix-excl", posixIoFinder ), +#if OS_VXWORKS + UNIXVFS("unix-namedsem", semIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-posix", posixIoFinder ), +#if !OS_VXWORKS + UNIXVFS("unix-flock", flockIoFinder ), +#endif +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix-afp", afpIoFinder ), + UNIXVFS("unix-nfs", nfsIoFinder ), + UNIXVFS("unix-proxy", proxyIoFinder ), +#endif + }; + unsigned int i; /* Loop counter */ - /* Did we just have a reader lock? */ - else if (pFile->local.nReaders){ - assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE || nNumberOfBytesToUnlockLow == 1); - pFile->local.nReaders --; - if (pFile->local.nReaders == 0) - { - pFile->shared->nReaders --; - } - bReturn = TRUE; - } - } + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==16 ); - /* Releasing a pending lock */ - else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToUnlockLow == 1){ - if (pFile->local.bPending){ - pFile->local.bPending = FALSE; - pFile->shared->bPending = FALSE; - bReturn = TRUE; - } - } - /* Releasing a reserved lock */ - else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToUnlockLow == 1){ - if (pFile->local.bReserved) { - pFile->local.bReserved = FALSE; - pFile->shared->bReserved = FALSE; - bReturn = TRUE; - } + /* Register all VFSes defined in the aVfs[] array */ + for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ + sqlite3_vfs_register(&aVfs[i], i==0); } + return SQLITE_OK; +} - winceMutexRelease(pFile->hMutex); - return bReturn; +/* +** Shutdown the operating system interface. +** +** Some operating systems might need to do some cleanup in this routine, +** to release dynamically allocated objects. But not on unix. +** This routine is a no-op for unix. +*/ +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; } + +#endif /* SQLITE_OS_UNIX */ +/************** End of os_unix.c *********************************************/ +/************** Begin file os_win.c ******************************************/ /* -** An implementation of the LockFileEx() API of windows for wince +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to windows. */ -static BOOL winceLockFileEx( - HANDLE *phFile, - DWORD dwFlags, - DWORD dwReserved, - DWORD nNumberOfBytesToLockLow, - DWORD nNumberOfBytesToLockHigh, - LPOVERLAPPED lpOverlapped -){ - UNUSED_PARAMETER(dwReserved); - UNUSED_PARAMETER(nNumberOfBytesToLockHigh); +#if SQLITE_OS_WIN /* This file is used for windows only */ + - /* If the caller wants a shared read lock, forward this call - ** to winceLockFile */ - if (lpOverlapped->Offset == (DWORD)SHARED_FIRST && - dwFlags == 1 && - nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ - return winceLockFile(phFile, SHARED_FIRST, 0, 1, 0); - } - return FALSE; -} /* -** End of the special code for wince -*****************************************************************************/ -#endif /* SQLITE_OS_WINCE */ +** A Note About Memory Allocation: +** +** This driver uses malloc()/free() directly rather than going through +** the SQLite-wrappers sqlite3_malloc()/sqlite3_free(). Those wrappers +** are designed for use on embedded systems where memory is scarce and +** malloc failures happen frequently. Win32 does not typically run on +** embedded systems, and when it does the developers normally have bigger +** problems to worry about than running out of memory. So there is not +** a compelling need to use the wrappers. +** +** But there is a good reason to not use the wrappers. If we use the +** wrappers then we will get simulated malloc() failures within this +** driver. And that causes all kinds of problems for our tests. We +** could enhance SQLite to deal with simulated malloc failures within +** the OS driver, but the code to deal with those failure would not +** be exercised on Linux (which does not need to malloc() in the driver) +** and so we would have difficulty writing coverage tests for that +** code. Better to leave the code out, we think. +** +** The point of this discussion is as follows: When creating a new +** OS layer for an embedded system, if you use this file as an example, +** avoid the use of malloc()/free(). Those routines work ok on windows +** desktops but not so well in embedded systems. +*/ -/***************************************************************************** -** The next group of routines implement the I/O methods specified -** by the sqlite3_io_methods object. -******************************************************************************/ +#include + +#ifdef __CYGWIN__ +# include +#endif /* -** Some microsoft compilers lack this definition. +** Macros used to determine whether or not to use threads. */ -#ifndef INVALID_SET_FILE_POINTER -# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#if defined(THREADSAFE) && THREADSAFE +# define SQLITE_W32_THREADS 1 #endif /* -** Move the current position of the file handle passed as the first -** argument to offset iOffset within the file. If successful, return 0. -** Otherwise, set pFile->lastErrno and return non-zero. +** Include code that is common to all os_*.c files */ -static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ - LONG upperBits; /* Most sig. 32 bits of new offset */ - LONG lowerBits; /* Least sig. 32 bits of new offset */ - DWORD dwRet; /* Value returned by SetFilePointer() */ +/************** Include os_common.h in the middle of os_win.c ****************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ - upperBits = (LONG)((iOffset>>32) & 0x7fffffff); - lowerBits = (LONG)(iOffset & 0xffffffff); +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif - /* API oddity: If successful, SetFilePointer() returns a dword - ** containing the lower 32-bits of the new file-offset. Or, if it fails, - ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, - ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occured, it is also necessary to call - ** GetLastError(). - */ - dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ - pFile->lastErrno = GetLastError(); - return 1; - } +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3OSTrace = 0; +#define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +#else +#define OSTRACE(X) +#endif - return 0; -} +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of os_common.h ****************/ +/************** Begin file hwtime.h ******************************************/ /* -** Close a file. +** 2008 May 27 ** -** It is reported that an attempt to close a handle might sometimes -** fail. This is a very unreasonable result, but windows is notorious -** for being unreasonable so I do not doubt that it might happen. If -** the close fails, we pause for 100 milliseconds and try again. As -** many as MX_CLOSE_ATTEMPT attempts to close the handle are made before -** giving up and returning an error. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. */ -#define MX_CLOSE_ATTEMPT 3 -static int winClose(sqlite3_file *id){ - int rc, cnt = 0; - winFile *pFile = (winFile*)id; - - assert( id!=0 ); - assert( pFile->pShm==0 ); - OSTRACE(("CLOSE %d\n", pFile->h)); - do{ - rc = CloseHandle(pFile->h); - /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ - }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (Sleep(100), 1) ); -#if SQLITE_OS_WINCE -#define WINCE_DELETION_ATTEMPTS 3 - winceDestroyLock(pFile); - if( pFile->zDeleteOnClose ){ - int cnt = 0; - while( - DeleteFileW(pFile->zDeleteOnClose)==0 - && GetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff - && cnt++ < WINCE_DELETION_ATTEMPTS - ){ - Sleep(100); /* Wait a little before trying again */ - } - free(pFile->zDeleteOnClose); - } -#endif - OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); - OpenCounter(-1); - return rc ? SQLITE_OK : SQLITE_IOERR; -} +#ifndef _HWTIME_H_ +#define _HWTIME_H_ /* -** Read data from a file into a buffer. Return SQLITE_OK if all -** bytes were read successfully and SQLITE_IOERR if anything goes -** wrong. +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. */ -static int winRead( - sqlite3_file *id, /* File to read from */ - void *pBuf, /* Write content into this buffer */ - int amt, /* Number of bytes to read */ - sqlite3_int64 offset /* Begin reading at this offset */ -){ - winFile *pFile = (winFile*)id; /* file handle */ - DWORD nRead; /* Number of bytes actually read from file */ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) - assert( id!=0 ); - SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + #if defined(__GNUC__) - if( seekWinFile(pFile, offset) ){ - return SQLITE_FULL; - } - if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ - pFile->lastErrno = GetLastError(); - return SQLITE_IOERR_READ; - } - if( nRead<(DWORD)amt ){ - /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pBuf)[nRead], 0, amt-nRead); - return SQLITE_IOERR_SHORT_READ; + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; } - return SQLITE_OK; -} + #elif defined(_MSC_VER) -/* -** Write data from a buffer into a file. Return SQLITE_OK on success -** or some other error code on failure. -*/ -static int winWrite( - sqlite3_file *id, /* File to write into */ - const void *pBuf, /* The bytes to be written */ - int amt, /* Number of bytes to write */ - sqlite3_int64 offset /* Offset into the file to begin writing at */ -){ - int rc; /* True if error has occured, else false */ - winFile *pFile = (winFile*)id; /* File handle */ + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } - assert( amt>0 ); - assert( pFile ); - SimulateIOError(return SQLITE_IOERR_WRITE); - SimulateDiskfullError(return SQLITE_FULL); + #endif - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#elif (defined(__GNUC__) && defined(__x86_64__)) - rc = seekWinFile(pFile, offset); - if( rc==0 ){ - u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ - int nRem = amt; /* Number of bytes yet to be written */ - DWORD nWrite; /* Bytes written by each WriteFile() call */ - - while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ - aRem += nWrite; - nRem -= nWrite; - } - if( nRem>0 ){ - pFile->lastErrno = GetLastError(); - rc = 1; - } + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; } + +#elif (defined(__GNUC__) && defined(__ppc__)) - if( rc ){ - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ - return SQLITE_FULL; - } - return SQLITE_IOERR_WRITE; + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; } - return SQLITE_OK; + +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(_HWTIME_H_) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in os_common.h ******************/ + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +SQLITE_API int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +SQLITE_API int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +SQLITE_API int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +SQLITE_API int sqlite3_io_error_benign = 0; /* True if errors are benign */ +SQLITE_API int sqlite3_diskfull_pending = 0; +SQLITE_API int sqlite3_diskfull = 0; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; } +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif /* -** Truncate an open file to a specified size +** When testing, keep a count of the number of open files. */ -static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ - winFile *pFile = (winFile*)id; /* File handle object */ - int rc = SQLITE_OK; /* Return code for this function */ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_open_file_count = 0; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif - assert( pFile ); +#endif /* !defined(_OS_COMMON_H_) */ - OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); - SimulateIOError(return SQLITE_IOERR_TRUNCATE); +/************** End of os_common.h *******************************************/ +/************** Continuing where we left off in os_win.c *********************/ - /* If the user has configured a chunk-size for this file, truncate the - ** file so that it consists of an integer number of chunks (i.e. the - ** actual file size after the operation may be larger than the requested - ** size). - */ - if( pFile->szChunk ){ - nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; - } +/* +** Some microsoft compilers lack this definition. +*/ +#ifndef INVALID_FILE_ATTRIBUTES +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +#endif - /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ - if( seekWinFile(pFile, nByte) ){ - rc = SQLITE_IOERR_TRUNCATE; - }else if( 0==SetEndOfFile(pFile->h) ){ - pFile->lastErrno = GetLastError(); - rc = SQLITE_IOERR_TRUNCATE; - } +/* +** Determine if we are dealing with WindowsCE - which has a much +** reduced API. +*/ +#if SQLITE_OS_WINCE +# define AreFileApisANSI() 1 +# define FormatMessageW(a,b,c,d,e,f,g) 0 +#endif - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); - return rc; -} +/* Forward references */ +typedef struct winShm winShm; /* A connection to shared-memory */ +typedef struct winShmNode winShmNode; /* A region of shared-memory */ -#ifdef SQLITE_TEST /* -** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occuring at the right times. +** WinCE lacks native support for file locking so we have to fake it +** with some code of our own. */ -SQLITE_API int sqlite3_sync_count = 0; -SQLITE_API int sqlite3_fullsync_count = 0; +#if SQLITE_OS_WINCE +typedef struct winceLock { + int nReaders; /* Number of reader locks obtained */ + BOOL bPending; /* Indicates a pending lock has been obtained */ + BOOL bReserved; /* Indicates a reserved lock has been obtained */ + BOOL bExclusive; /* Indicates an exclusive lock has been obtained */ +} winceLock; #endif /* -** Make sure all writes to a particular file are committed to disk. +** The winFile structure is a subclass of sqlite3_file* specific to the win32 +** portability layer. */ -static int winSync(sqlite3_file *id, int flags){ -#if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || defined(SQLITE_DEBUG) - winFile *pFile = (winFile*)id; -#else - UNUSED_PARAMETER(id); +typedef struct winFile winFile; +struct winFile { + const sqlite3_io_methods *pMethod; /*** Must be first ***/ + sqlite3_vfs *pVfs; /* The VFS used to open this file */ + HANDLE h; /* Handle for accessing the file */ + unsigned char locktype; /* Type of lock currently held on this file */ + short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + DWORD lastErrno; /* The Windows errno from the last I/O error */ + DWORD sectorSize; /* Sector size of the device file is on */ + winShm *pShm; /* Instance of shared memory on this file */ + const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_OS_WINCE + WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ + HANDLE hMutex; /* Mutex used to control access to shared lock */ + HANDLE hShared; /* Shared memory segment used for locking */ + winceLock local; /* Locks obtained by this instance of winFile */ + winceLock *shared; /* Global shared lock memory for the file */ #endif +}; - assert( pFile ); - /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ - assert((flags&0x0F)==SQLITE_SYNC_NORMAL - || (flags&0x0F)==SQLITE_SYNC_FULL - ); - OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); +/* +** Forward prototypes. +*/ +static int getSectorSize( + sqlite3_vfs *pVfs, + const char *zRelative /* UTF-8 file name */ +); -#ifndef SQLITE_TEST - UNUSED_PARAMETER(flags); +/* +** The following variable is (normally) set once and never changes +** thereafter. It records whether the operating system is Win95 +** or WinNT. +** +** 0: Operating system unknown. +** 1: Operating system is Win95. +** 2: Operating system is WinNT. +** +** In order to facilitate testing on a WinNT system, the test fixture +** can manually set this value to 1 to emulate Win98 behavior. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_os_type = 0; #else - if( flags & SQLITE_SYNC_FULL ){ - sqlite3_fullsync_count++; - } - sqlite3_sync_count++; +static int sqlite3_os_type = 0; #endif - /* Unix cannot, but some systems may return SQLITE_FULL from here. This - ** line is to test that doing so does not cause any problems. - */ - SimulateDiskfullError( return SQLITE_FULL ); - SimulateIOError( return SQLITE_IOERR; ); - - /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a - ** no-op - */ -#ifdef SQLITE_NO_SYNC - return SQLITE_OK; +/* +** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, +** or WinCE. Return false (zero) for Win95, Win98, or WinME. +** +** Here is an interesting observation: Win95, Win98, and WinME lack +** the LockFileEx() API. But we can still statically link against that +** API as long as we don't call it when running Win95/98/ME. A call to +** this routine is used to determine if the host is Win95/98/ME or +** WinNT/2K/XP so that we will know whether or not we can safely call +** the LockFileEx() API. +*/ +#if SQLITE_OS_WINCE +# define isNT() (1) #else - if( FlushFileBuffers(pFile->h) ){ - return SQLITE_OK; - }else{ - pFile->lastErrno = GetLastError(); - return SQLITE_IOERR; + static int isNT(void){ + if( sqlite3_os_type==0 ){ + OSVERSIONINFO sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + GetVersionEx(&sInfo); + sqlite3_os_type = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; + } + return sqlite3_os_type==2; } -#endif +#endif /* SQLITE_OS_WINCE */ + +/* +** Convert a UTF-8 string to microsoft unicode (UTF-16?). +** +** Space to hold the returned string is obtained from malloc. +*/ +static WCHAR *utf8ToUnicode(const char *zFilename){ + int nChar; + WCHAR *zWideFilename; + + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + zWideFilename = malloc( nChar*sizeof(zWideFilename[0]) ); + if( zWideFilename==0 ){ + return 0; + } + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, nChar); + if( nChar==0 ){ + free(zWideFilename); + zWideFilename = 0; + } + return zWideFilename; } /* -** Determine the current size of a file in bytes +** Convert microsoft unicode to UTF-8. Space to hold the returned string is +** obtained from malloc(). */ -static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ - DWORD upperBits; - DWORD lowerBits; - winFile *pFile = (winFile*)id; - DWORD error; +static char *unicodeToUtf8(const WCHAR *zWideFilename){ + int nByte; + char *zFilename; - assert( id!=0 ); - SimulateIOError(return SQLITE_IOERR_FSTAT); - lowerBits = GetFileSize(pFile->h, &upperBits); - if( (lowerBits == INVALID_FILE_SIZE) - && ((error = GetLastError()) != NO_ERROR) ) - { - pFile->lastErrno = error; - return SQLITE_IOERR_FSTAT; + nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); + zFilename = malloc( nByte ); + if( zFilename==0 ){ + return 0; } - *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; - return SQLITE_OK; + nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, + 0, 0); + if( nByte == 0 ){ + free(zFilename); + zFilename = 0; + } + return zFilename; } /* -** LOCKFILE_FAIL_IMMEDIATELY is undefined on some Windows systems. +** Convert an ansi string to microsoft unicode, based on the +** current codepage settings for file apis. +** +** Space to hold the returned string is obtained +** from malloc. */ -#ifndef LOCKFILE_FAIL_IMMEDIATELY -# define LOCKFILE_FAIL_IMMEDIATELY 1 -#endif +static WCHAR *mbcsToUnicode(const char *zFilename){ + int nByte; + WCHAR *zMbcsFilename; + int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; + + nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, NULL,0)*sizeof(WCHAR); + zMbcsFilename = malloc( nByte*sizeof(zMbcsFilename[0]) ); + if( zMbcsFilename==0 ){ + return 0; + } + nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, nByte); + if( nByte==0 ){ + free(zMbcsFilename); + zMbcsFilename = 0; + } + return zMbcsFilename; +} /* -** Acquire a reader lock. -** Different API routines are called depending on whether or not this -** is Win95 or WinNT. +** Convert microsoft unicode to multibyte character string, based on the +** user's Ansi codepage. +** +** Space to hold the returned string is obtained from +** malloc(). */ -static int getReadLock(winFile *pFile){ - int res; - if( isNT() ){ - OVERLAPPED ovlp; - ovlp.Offset = SHARED_FIRST; - ovlp.OffsetHigh = 0; - ovlp.hEvent = 0; - res = LockFileEx(pFile->h, LOCKFILE_FAIL_IMMEDIATELY, - 0, SHARED_SIZE, 0, &ovlp); -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +static char *unicodeToMbcs(const WCHAR *zWideFilename){ + int nByte; + char *zFilename; + int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; + + nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); + zFilename = malloc( nByte ); + if( zFilename==0 ){ + return 0; + } + nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, nByte, + 0, 0); + if( nByte == 0 ){ + free(zFilename); + zFilename = 0; + } + return zFilename; +} + +/* +** Convert multibyte character string to UTF-8. Space to hold the +** returned string is obtained from malloc(). */ -#if SQLITE_OS_WINCE==0 - }else{ - int lk; - sqlite3_randomness(sizeof(lk), &lk); - pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); - res = LockFile(pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); -#endif +SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ + char *zFilenameUtf8; + WCHAR *zTmpWide; + + zTmpWide = mbcsToUnicode(zFilename); + if( zTmpWide==0 ){ + return 0; } - if( res == 0 ){ - pFile->lastErrno = GetLastError(); + zFilenameUtf8 = unicodeToUtf8(zTmpWide); + free(zTmpWide); + return zFilenameUtf8; +} + +/* +** Convert UTF-8 to multibyte character string. Space to hold the +** returned string is obtained from malloc(). +*/ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ + char *zFilenameMbcs; + WCHAR *zTmpWide; + + zTmpWide = utf8ToUnicode(zFilename); + if( zTmpWide==0 ){ + return 0; } - return res; + zFilenameMbcs = unicodeToMbcs(zTmpWide); + free(zTmpWide); + return zFilenameMbcs; } + /* -** Undo a readlock +** The return value of getLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). */ -static int unlockReadLock(winFile *pFile){ - int res; +static int getLastErrorMsg(int nBuf, char *zBuf){ + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + DWORD error = GetLastError(); + DWORD dwLen = 0; + char *zOut = 0; + if( isNT() ){ - res = UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + WCHAR *zTempWide = NULL; + dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + error, + 0, + (LPWSTR) &zTempWide, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + zOut = unicodeToUtf8(zTempWide); + /* free the system buffer allocated by FormatMessage */ + LocalFree(zTempWide); + } /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - res = UnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); + char *zTemp = NULL; + dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + error, + 0, + (LPSTR) &zTemp, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + /* free the system buffer allocated by FormatMessage */ + LocalFree(zTemp); + } #endif } - if( res == 0 ){ - pFile->lastErrno = GetLastError(); + if( 0 == dwLen ){ + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); + }else{ + /* copy a maximum of nBuf chars to output buffer */ + sqlite3_snprintf(nBuf, zBuf, "%s", zOut); + /* free the UTF8 buffer */ + free(zOut); } - return res; + return 0; } /* -** Lock the file with the lock specified by parameter locktype - one -** of the following: -** -** (1) SHARED_LOCK -** (2) RESERVED_LOCK -** (3) PENDING_LOCK -** (4) EXCLUSIVE_LOCK ** -** Sometimes when requesting one lock state, additional lock states -** are inserted in between. The locking might fail on one of the later -** transitions leaving the lock state different from what it started but -** still short of its goal. The following chart shows the allowed -** transitions and the inserted intermediate states: +** This function - winLogErrorAtLine() - is only ever called via the macro +** winLogError(). ** -** UNLOCKED -> SHARED -** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE -** RESERVED -> (PENDING) -> EXCLUSIVE -** PENDING -> EXCLUSIVE +** This routine is invoked after an error occurs in an OS function. +** It logs a message using sqlite3_log() containing the current value of +** error code and, if possible, the human-readable equivalent from +** FormatMessage. ** -** This routine will only increase a lock. The winUnlock() routine -** erases all locks at once and returns us immediately to locking level 0. -** It is not possible to lower the locking level one step at a time. You -** must go straight to locking level 0. +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed and the the associated file-system path, if any. */ -static int winLock(sqlite3_file *id, int locktype){ - int rc = SQLITE_OK; /* Return code from subroutines */ - int res = 1; /* Result of a windows lock call */ - int newLocktype; /* Set pFile->locktype to this value before exiting */ - int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ - winFile *pFile = (winFile*)id; - DWORD error = NO_ERROR; +#define winLogError(a,b,c) winLogErrorAtLine(a,b,c,__LINE__) +static int winLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char zMsg[500]; /* Human readable error text */ + int i; /* Loop counter */ + DWORD iErrno = GetLastError(); /* Error code */ + + zMsg[0] = 0; + getLastErrorMsg(sizeof(zMsg), zMsg); + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} + zMsg[i] = 0; + sqlite3_log(errcode, + "os_win.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zMsg + ); - assert( id!=0 ); - OSTRACE(("LOCK %d %d was %d(%d)\n", - pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); + return errcode; +} - /* If there is already a lock of this type or more restrictive on the - ** OsFile, do nothing. Don't use the end_lock: exit path, as - ** sqlite3OsEnterMutex() hasn't been called yet. - */ - if( pFile->locktype>=locktype ){ - return SQLITE_OK; - } +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ +/* +** WindowsCE does not have a localtime() function. So create a +** substitute. +*/ +struct tm *__cdecl localtime(const time_t *t) +{ + static struct tm y; + FILETIME uTm, lTm; + SYSTEMTIME pTm; + sqlite3_int64 t64; + t64 = *t; + t64 = (t64 + 11644473600)*10000000; + uTm.dwLowDateTime = (DWORD)(t64 & 0xFFFFFFFF); + uTm.dwHighDateTime= (DWORD)(t64 >> 32); + FileTimeToLocalFileTime(&uTm,&lTm); + FileTimeToSystemTime(&lTm,&pTm); + y.tm_year = pTm.wYear - 1900; + y.tm_mon = pTm.wMonth - 1; + y.tm_wday = pTm.wDayOfWeek; + y.tm_mday = pTm.wDay; + y.tm_hour = pTm.wHour; + y.tm_min = pTm.wMinute; + y.tm_sec = pTm.wSecond; + return &y; +} - /* Make sure the locking sequence is correct - */ - assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); - assert( locktype!=PENDING_LOCK ); - assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); +/* This will never be called, but defined to make the code compile */ +#define GetTempPathA(a,b) - /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or - ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of - ** the PENDING_LOCK byte is temporary. - */ - newLocktype = pFile->locktype; - if( (pFile->locktype==NO_LOCK) - || ( (locktype==EXCLUSIVE_LOCK) - && (pFile->locktype==RESERVED_LOCK)) - ){ - int cnt = 3; - while( cnt-->0 && (res = LockFile(pFile->h, PENDING_BYTE, 0, 1, 0))==0 ){ - /* Try 3 times to get the pending lock. The pending lock might be - ** held by another reader process who will release it momentarily. - */ - OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); - Sleep(1); - } - gotPendingLock = res; - if( !res ){ - error = GetLastError(); - } - } +#define LockFile(a,b,c,d,e) winceLockFile(&a, b, c, d, e) +#define UnlockFile(a,b,c,d,e) winceUnlockFile(&a, b, c, d, e) +#define LockFileEx(a,b,c,d,e,f) winceLockFileEx(&a, b, c, d, e, f) - /* Acquire a shared lock - */ - if( locktype==SHARED_LOCK && res ){ - assert( pFile->locktype==NO_LOCK ); - res = getReadLock(pFile); - if( res ){ - newLocktype = SHARED_LOCK; - }else{ - error = GetLastError(); - } +#define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] + +/* +** Acquire a lock on the handle h +*/ +static void winceMutexAcquire(HANDLE h){ + DWORD dwErr; + do { + dwErr = WaitForSingleObject(h, INFINITE); + } while (dwErr != WAIT_OBJECT_0 && dwErr != WAIT_ABANDONED); +} +/* +** Release a lock acquired by winceMutexAcquire() +*/ +#define winceMutexRelease(h) ReleaseMutex(h) + +/* +** Create the mutex and shared memory used for locking in the file +** descriptor pFile +*/ +static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ + WCHAR *zTok; + WCHAR *zName = utf8ToUnicode(zFilename); + BOOL bInit = TRUE; + + /* Initialize the local lockdata */ + ZeroMemory(&pFile->local, sizeof(pFile->local)); + + /* Replace the backslashes from the filename and lowercase it + ** to derive a mutex name. */ + zTok = CharLowerW(zName); + for (;*zTok;zTok++){ + if (*zTok == '\\') *zTok = '_'; } - /* Acquire a RESERVED lock - */ - if( locktype==RESERVED_LOCK && res ){ - assert( pFile->locktype==SHARED_LOCK ); - res = LockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); - if( res ){ - newLocktype = RESERVED_LOCK; - }else{ - error = GetLastError(); - } + /* Create/open the named mutex */ + pFile->hMutex = CreateMutexW(NULL, FALSE, zName); + if (!pFile->hMutex){ + pFile->lastErrno = GetLastError(); + winLogError(SQLITE_ERROR, "winceCreateLock1", zFilename); + free(zName); + return FALSE; } - /* Acquire a PENDING lock + /* Acquire the mutex before continuing */ + winceMutexAcquire(pFile->hMutex); + + /* Since the names of named mutexes, semaphores, file mappings etc are + ** case-sensitive, take advantage of that by uppercasing the mutex name + ** and using that as the shared filemapping name. */ - if( locktype==EXCLUSIVE_LOCK && res ){ - newLocktype = PENDING_LOCK; - gotPendingLock = 0; + CharUpperW(zName); + pFile->hShared = CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE, 0, sizeof(winceLock), + zName); + + /* Set a flag that indicates we're the first to create the memory so it + ** must be zero-initialized */ + if (GetLastError() == ERROR_ALREADY_EXISTS){ + bInit = FALSE; } - /* Acquire an EXCLUSIVE lock - */ - if( locktype==EXCLUSIVE_LOCK && res ){ - assert( pFile->locktype>=SHARED_LOCK ); - res = unlockReadLock(pFile); - OSTRACE(("unreadlock = %d\n", res)); - res = LockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); - if( res ){ - newLocktype = EXCLUSIVE_LOCK; - }else{ - error = GetLastError(); - OSTRACE(("error-code = %d\n", error)); - getReadLock(pFile); + free(zName); + + /* If we succeeded in making the shared memory handle, map it. */ + if (pFile->hShared){ + pFile->shared = (winceLock*)MapViewOfFile(pFile->hShared, + FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); + /* If mapping failed, close the shared memory handle and erase it */ + if (!pFile->shared){ + pFile->lastErrno = GetLastError(); + winLogError(SQLITE_ERROR, "winceCreateLock2", zFilename); + CloseHandle(pFile->hShared); + pFile->hShared = NULL; } } - /* If we are holding a PENDING lock that ought to be released, then - ** release it now. - */ - if( gotPendingLock && locktype==SHARED_LOCK ){ - UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); + /* If shared memory could not be created, then close the mutex and fail */ + if (pFile->hShared == NULL){ + winceMutexRelease(pFile->hMutex); + CloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + return FALSE; } - - /* Update the state of the lock has held in the file descriptor then - ** return the appropriate result code. - */ - if( res ){ - rc = SQLITE_OK; - }else{ - OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype)); - pFile->lastErrno = error; - rc = SQLITE_BUSY; + + /* Initialize the shared memory if we're supposed to */ + if (bInit) { + ZeroMemory(pFile->shared, sizeof(winceLock)); } - pFile->locktype = (u8)newLocktype; - return rc; + + winceMutexRelease(pFile->hMutex); + return TRUE; } /* -** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, return -** non-zero, otherwise zero. +** Destroy the part of winFile that deals with wince locks */ -static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc; - winFile *pFile = (winFile*)id; - - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); +static void winceDestroyLock(winFile *pFile){ + if (pFile->hMutex){ + /* Acquire the mutex */ + winceMutexAcquire(pFile->hMutex); - assert( id!=0 ); - if( pFile->locktype>=RESERVED_LOCK ){ - rc = 1; - OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); - }else{ - rc = LockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); - if( rc ){ - UnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + /* The following blocks should probably assert in debug mode, but they + are to cleanup in case any locks remained open */ + if (pFile->local.nReaders){ + pFile->shared->nReaders --; } - rc = !rc; - OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); + if (pFile->local.bReserved){ + pFile->shared->bReserved = FALSE; + } + if (pFile->local.bPending){ + pFile->shared->bPending = FALSE; + } + if (pFile->local.bExclusive){ + pFile->shared->bExclusive = FALSE; + } + + /* De-reference and close our copy of the shared memory handle */ + UnmapViewOfFile(pFile->shared); + CloseHandle(pFile->hShared); + + /* Done with the mutex */ + winceMutexRelease(pFile->hMutex); + CloseHandle(pFile->hMutex); + pFile->hMutex = NULL; } - *pResOut = rc; - return SQLITE_OK; } -/* -** Lower the locking level on file descriptor id to locktype. locktype -** must be either NO_LOCK or SHARED_LOCK. -** -** If the locking level of the file descriptor is already at or below -** the requested locking level, this routine is a no-op. -** -** It is not possible for this routine to fail if the second argument -** is NO_LOCK. If the second argument is SHARED_LOCK then this routine -** might return SQLITE_IOERR; +/* +** An implementation of the LockFile() API of windows for wince */ -static int winUnlock(sqlite3_file *id, int locktype){ - int type; - winFile *pFile = (winFile*)id; - int rc = SQLITE_OK; - assert( pFile!=0 ); - assert( locktype<=SHARED_LOCK ); - OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, - pFile->locktype, pFile->sharedLockByte)); - type = pFile->locktype; - if( type>=EXCLUSIVE_LOCK ){ - UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); - if( locktype==SHARED_LOCK && !getReadLock(pFile) ){ - /* This should never happen. We should always be able to - ** reacquire the read lock */ - rc = SQLITE_IOERR_UNLOCK; +static BOOL winceLockFile( + HANDLE *phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToLockLow, + DWORD nNumberOfBytesToLockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToLockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Wanting an exclusive lock? */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST + && nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ + if (pFile->shared->nReaders == 0 && pFile->shared->bExclusive == 0){ + pFile->shared->bExclusive = TRUE; + pFile->local.bExclusive = TRUE; + bReturn = TRUE; } } - if( type>=RESERVED_LOCK ){ - UnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + + /* Want a read-only lock? */ + else if (dwFileOffsetLow == (DWORD)SHARED_FIRST && + nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bExclusive == 0){ + pFile->local.nReaders ++; + if (pFile->local.nReaders == 1){ + pFile->shared->nReaders ++; + } + bReturn = TRUE; + } } - if( locktype==NO_LOCK && type>=SHARED_LOCK ){ - unlockReadLock(pFile); + + /* Want a pending lock? */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToLockLow == 1){ + /* If no pending lock has been acquired, then acquire it */ + if (pFile->shared->bPending == 0) { + pFile->shared->bPending = TRUE; + pFile->local.bPending = TRUE; + bReturn = TRUE; + } } - if( type>=PENDING_LOCK ){ - UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); + + /* Want a reserved lock? */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bReserved == 0) { + pFile->shared->bReserved = TRUE; + pFile->local.bReserved = TRUE; + bReturn = TRUE; + } } - pFile->locktype = (u8)locktype; - return rc; + + winceMutexRelease(pFile->hMutex); + return bReturn; } /* -** Control and query of the open file handle. +** An implementation of the UnlockFile API of windows for wince */ -static int winFileControl(sqlite3_file *id, int op, void *pArg){ - switch( op ){ - case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((winFile*)id)->locktype; - return SQLITE_OK; +static BOOL winceUnlockFile( + HANDLE *phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToUnlockLow, + DWORD nNumberOfBytesToUnlockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToUnlockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Releasing a reader lock or an exclusive lock */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST){ + /* Did we have an exclusive lock? */ + if (pFile->local.bExclusive){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE); + pFile->local.bExclusive = FALSE; + pFile->shared->bExclusive = FALSE; + bReturn = TRUE; } - case SQLITE_LAST_ERRNO: { - *(int*)pArg = (int)((winFile*)id)->lastErrno; - return SQLITE_OK; + + /* Did we just have a reader lock? */ + else if (pFile->local.nReaders){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE || nNumberOfBytesToUnlockLow == 1); + pFile->local.nReaders --; + if (pFile->local.nReaders == 0) + { + pFile->shared->nReaders --; + } + bReturn = TRUE; } - case SQLITE_FCNTL_CHUNK_SIZE: { - ((winFile*)id)->szChunk = *(int *)pArg; - return SQLITE_OK; + } + + /* Releasing a pending lock */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bPending){ + pFile->local.bPending = FALSE; + pFile->shared->bPending = FALSE; + bReturn = TRUE; } - case SQLITE_FCNTL_SIZE_HINT: { - sqlite3_int64 sz = *(sqlite3_int64*)pArg; - SimulateIOErrorBenign(1); - winTruncate(id, sz); - SimulateIOErrorBenign(0); - return SQLITE_OK; + } + /* Releasing a reserved lock */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bReserved) { + pFile->local.bReserved = FALSE; + pFile->shared->bReserved = FALSE; + bReturn = TRUE; } } - return SQLITE_ERROR; -} -/* -** Return the sector size in bytes of the underlying block device for -** the specified file. This is almost always 512 bytes, but may be -** larger for some devices. -** -** SQLite code assumes this function cannot fail. It also assumes that -** if two files are created in the same file-system directory (i.e. -** a database and its journal file) that the sector size will be the -** same for both. -*/ -static int winSectorSize(sqlite3_file *id){ - assert( id!=0 ); - return (int)(((winFile*)id)->sectorSize); + winceMutexRelease(pFile->hMutex); + return bReturn; } /* -** Return a vector of device characteristics. +** An implementation of the LockFileEx() API of windows for wince */ -static int winDeviceCharacteristics(sqlite3_file *id){ - UNUSED_PARAMETER(id); - return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; +static BOOL winceLockFileEx( + HANDLE *phFile, + DWORD dwFlags, + DWORD dwReserved, + DWORD nNumberOfBytesToLockLow, + DWORD nNumberOfBytesToLockHigh, + LPOVERLAPPED lpOverlapped +){ + UNUSED_PARAMETER(dwReserved); + UNUSED_PARAMETER(nNumberOfBytesToLockHigh); + + /* If the caller wants a shared read lock, forward this call + ** to winceLockFile */ + if (lpOverlapped->Offset == (DWORD)SHARED_FIRST && + dwFlags == 1 && + nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ + return winceLockFile(phFile, SHARED_FIRST, 0, 1, 0); + } + return FALSE; } +/* +** End of the special code for wince +*****************************************************************************/ +#endif /* SQLITE_OS_WINCE */ -#ifndef SQLITE_OMIT_WAL +/***************************************************************************** +** The next group of routines implement the I/O methods specified +** by the sqlite3_io_methods object. +******************************************************************************/ -/* -** Windows will only let you create file view mappings -** on allocation size granularity boundaries. -** During sqlite3_os_init() we do a GetSystemInfo() -** to get the granularity size. +/* +** Some microsoft compilers lack this definition. */ -SYSTEM_INFO winSysInfo; +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif /* -** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the winLockInfo objects used by -** this file, all of which may be shared by multiple threads. -** -** Function winShmMutexHeld() is used to assert() that the global mutex -** is held when required. This function is only used as part of assert() -** statements. e.g. -** -** winShmEnterMutex() -** assert( winShmMutexHeld() ); -** winShmLeaveMutex() +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. */ -static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); -} -static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); -} -#ifdef SQLITE_DEBUG -static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occured, it is also necessary to call + ** GetLastError(). + */ + dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ + pFile->lastErrno = GetLastError(); + winLogError(SQLITE_IOERR_SEEK, "seekWinFile", pFile->zPath); + return 1; + } + + return 0; } -#endif /* -** Object used to represent a single file opened and mmapped to provide -** shared memory. When multiple threads all reference the same -** log-summary, each thread has its own winFile object, but they all -** point to a single instance of this object. In other words, each -** log-summary is opened only once per process. -** -** winShmMutexHeld() must be true when creating or destroying -** this object or while reading or writing the following fields: -** -** nRef -** pNext -** -** The following fields are read-only after the object is created: -** -** fid -** zFilename -** -** Either winShmNode.mutex must be held or winShmNode.nRef==0 and -** winShmMutexHeld() is true when reading or writing any other field -** in this structure. +** Close a file. ** +** It is reported that an attempt to close a handle might sometimes +** fail. This is a very unreasonable result, but windows is notorious +** for being unreasonable so I do not doubt that it might happen. If +** the close fails, we pause for 100 milliseconds and try again. As +** many as MX_CLOSE_ATTEMPT attempts to close the handle are made before +** giving up and returning an error. */ -struct winShmNode { - sqlite3_mutex *mutex; /* Mutex to access this object */ - char *zFilename; /* Name of the file */ - winFile hFile; /* File handle from winOpen */ - - int szRegion; /* Size of shared-memory regions */ - int nRegion; /* Size of array apRegion */ - struct ShmRegion { - HANDLE hMap; /* File handle from CreateFileMapping */ - void *pMap; - } *aRegion; - DWORD lastErrno; /* The Windows errno from the last I/O error */ +#define MX_CLOSE_ATTEMPT 3 +static int winClose(sqlite3_file *id){ + int rc, cnt = 0; + winFile *pFile = (winFile*)id; - int nRef; /* Number of winShm objects pointing to this */ - winShm *pFirst; /* All winShm objects pointing to this */ - winShmNode *pNext; /* Next in list of all winShmNode objects */ -#ifdef SQLITE_DEBUG - u8 nextShmId; /* Next available winShm.id value */ + assert( id!=0 ); + assert( pFile->pShm==0 ); + OSTRACE(("CLOSE %d\n", pFile->h)); + do{ + rc = CloseHandle(pFile->h); + /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (Sleep(100), 1) ); +#if SQLITE_OS_WINCE +#define WINCE_DELETION_ATTEMPTS 3 + winceDestroyLock(pFile); + if( pFile->zDeleteOnClose ){ + int cnt = 0; + while( + DeleteFileW(pFile->zDeleteOnClose)==0 + && GetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && cnt++ < WINCE_DELETION_ATTEMPTS + ){ + Sleep(100); /* Wait a little before trying again */ + } + free(pFile->zDeleteOnClose); + } #endif -}; + OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); + OpenCounter(-1); + return rc ? SQLITE_OK + : winLogError(SQLITE_IOERR_CLOSE, "winClose", pFile->zPath); +} /* -** A global array of all winShmNode objects. -** -** The winShmMutexHeld() must be true while reading or writing this list. +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. */ -static winShmNode *winShmNodeList = 0; +static int winRead( + sqlite3_file *id, /* File to read from */ + void *pBuf, /* Write content into this buffer */ + int amt, /* Number of bytes to read */ + sqlite3_int64 offset /* Begin reading at this offset */ +){ + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ -/* -** Structure used internally by this VFS to record the state of an -** open shared memory connection. -** -** The following fields are initialized when this object is created and -** are read-only thereafter: -** -** winShm.pShmNode -** winShm.id -** -** All other fields are read/write. The winShm.pShmNode->mutex must be held -** while accessing any read/write fields. -*/ -struct winShm { - winShmNode *pShmNode; /* The underlying winShmNode object */ - winShm *pNext; /* Next winShm with the same winShmNode */ - u8 hasMutex; /* True if holding the winShmNode mutex */ - u16 sharedMask; /* Mask of shared locks held */ - u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG - u8 id; /* Id of this connection with its winShmNode */ -#endif -}; + assert( id!=0 ); + SimulateIOError(return SQLITE_IOERR_READ); + OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); -/* -** Constants used for locking -*/ -#define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ -#define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + if( seekWinFile(pFile, offset) ){ + return SQLITE_FULL; + } + if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + pFile->lastErrno = GetLastError(); + return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); + } + if( nRead<(DWORD)amt ){ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[nRead], 0, amt-nRead); + return SQLITE_IOERR_SHORT_READ; + } + + return SQLITE_OK; +} /* -** Apply advisory locks for all n bytes beginning at ofst. +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. */ -#define _SHM_UNLCK 1 -#define _SHM_RDLCK 2 -#define _SHM_WRLCK 3 -static int winShmSystemLock( - winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ - int ofst, /* Offset to first byte to be locked/unlocked */ - int nByte /* Number of bytes to lock or unlock */ +static int winWrite( + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - OVERLAPPED ovlp; - DWORD dwFlags; - int rc = 0; /* Result code form Lock/UnlockFileEx() */ + int rc; /* True if error has occured, else false */ + winFile *pFile = (winFile*)id; /* File handle */ - /* Access to the winShmNode object is serialized by the caller */ - assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + assert( amt>0 ); + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_WRITE); + SimulateDiskfullError(return SQLITE_FULL); - /* Initialize the locking parameters */ - dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); - memset(&ovlp, 0, sizeof(OVERLAPPED)); - ovlp.Offset = ofst; + rc = seekWinFile(pFile, offset); + if( rc==0 ){ + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ - /* Release/Acquire the system-level lock */ - if( lockType==_SHM_UNLCK ){ - rc = UnlockFileEx(pFile->hFile.h, 0, nByte, 0, &ovlp); - }else{ - rc = LockFileEx(pFile->hFile.h, dwFlags, 0, nByte, 0, &ovlp); - } - - if( rc!= 0 ){ - rc = SQLITE_OK; - }else{ - pFile->lastErrno = GetLastError(); - rc = SQLITE_BUSY; + while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = GetLastError(); + rc = 1; + } } - OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", - pFile->hFile.h, - rc==SQLITE_OK ? "ok" : "failed", - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - pFile->lastErrno)); - - return rc; -} - -/* Forward references to VFS methods */ -static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); -static int winDelete(sqlite3_vfs *,const char*,int); - -/* -** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. -** -** This is not a VFS shared-memory method; it is a utility function called -** by VFS shared-memory methods. -*/ -static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ - winShmNode **pp; - winShmNode *p; - BOOL bRc; - assert( winShmMutexHeld() ); - pp = &winShmNodeList; - while( (p = *pp)!=0 ){ - if( p->nRef==0 ){ - int i; - if( p->mutex ) sqlite3_mutex_free(p->mutex); - for(i=0; inRegion; i++){ - bRc = UnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", - (int)GetCurrentProcessId(), i, - bRc ? "ok" : "failed")); - bRc = CloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", - (int)GetCurrentProcessId(), i, - bRc ? "ok" : "failed")); - } - if( p->hFile.h != INVALID_HANDLE_VALUE ){ - SimulateIOErrorBenign(1); - winClose((sqlite3_file *)&p->hFile); - SimulateIOErrorBenign(0); - } - if( deleteFlag ){ - SimulateIOErrorBenign(1); - winDelete(pVfs, p->zFilename, 0); - SimulateIOErrorBenign(0); - } - *pp = p->pNext; - sqlite3_free(p->aRegion); - sqlite3_free(p); - }else{ - pp = &p->pNext; + if( rc ){ + if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) + || ( pFile->lastErrno==ERROR_DISK_FULL )){ + return SQLITE_FULL; } + return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); } + return SQLITE_OK; } /* -** Open the shared-memory area associated with database file pDbFd. -** -** When opening a new shared-memory file, if no other instances of that -** file are currently open, in this process or in other processes, then -** the file must be truncated to zero length or have its header cleared. +** Truncate an open file to a specified size */ -static int winOpenSharedMemory(winFile *pDbFd){ - struct winShm *p; /* The connection to be opened */ - struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc; /* Result code */ - struct winShmNode *pNew; /* Newly allocated winShmNode */ - int nName; /* Size of zName in bytes */ +static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ - assert( pDbFd->pShm==0 ); /* Not previously opened */ + assert( pFile ); - /* Allocate space for the new sqlite3_shm object. Also speculatively - ** allocate space for a new winShmNode and filename. - */ - p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM; - memset(p, 0, sizeof(*p)); - nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3_malloc( sizeof(*pShmNode) + nName + 15 ); - if( pNew==0 ){ - sqlite3_free(p); - return SQLITE_NOMEM; - } - memset(pNew, 0, sizeof(*pNew)); - pNew->zFilename = (char*)&pNew[1]; - sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); + SimulateIOError(return SQLITE_IOERR_TRUNCATE); - /* Look to see if there is an existing winShmNode that can be used. - ** If no matching winShmNode currently exists, create a new one. + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). */ - winShmEnterMutex(); - for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ - /* TBD need to come up with better match here. Perhaps - ** use FILE_ID_BOTH_DIR_INFO Structure. - */ - if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } - if( pShmNode ){ - sqlite3_free(pNew); - }else{ - pShmNode = pNew; - pNew = 0; - ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; - pShmNode->pNext = winShmNodeList; - winShmNodeList = pShmNode; - - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; - } - - rc = winOpen(pDbFd->pVfs, - pShmNode->zFilename, /* Name of the file (UTF-8) */ - (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ - 0); - if( SQLITE_OK!=rc ){ - rc = SQLITE_CANTOPEN_BKPT; - goto shm_open_err; - } - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); - if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMOPEN; - } - } - if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( seekWinFile(pFile, nByte) ){ + rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate1", pFile->zPath); + }else if( 0==SetEndOfFile(pFile->h) ){ + pFile->lastErrno = GetLastError(); + rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate2", pFile->zPath); } - /* Make the new connection a child of the winShmNode */ - p->pShmNode = pShmNode; -#ifdef SQLITE_DEBUG - p->id = pShmNode->nextShmId++; -#endif - pShmNode->nRef++; - pDbFd->pShm = p; - winShmLeaveMutex(); - - /* The reference count on pShmNode has already been incremented under - ** the cover of the winShmEnterMutex() mutex and the pointer from the - ** new (struct winShm) object to the pShmNode has been set. All that is - ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. - */ - sqlite3_mutex_enter(pShmNode->mutex); - p->pNext = pShmNode->pFirst; - pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; - - /* Jump here on any error */ -shm_open_err: - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); - winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ - sqlite3_free(p); - sqlite3_free(pNew); - winShmLeaveMutex(); + OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; } +#ifdef SQLITE_TEST /* -** Close a connection to shared-memory. Delete the underlying -** storage if deleteFlag is true. +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occuring at the right times. */ -static int winShmUnmap( - sqlite3_file *fd, /* Database holding shared memory */ - int deleteFlag /* Delete after closing if true */ -){ - winFile *pDbFd; /* Database holding shared-memory */ - winShm *p; /* The connection to be closed */ - winShmNode *pShmNode; /* The underlying shared-memory file */ - winShm **pp; /* For looping over sibling connections */ +SQLITE_API int sqlite3_sync_count = 0; +SQLITE_API int sqlite3_fullsync_count = 0; +#endif - pDbFd = (winFile*)fd; - p = pDbFd->pShm; - if( p==0 ) return SQLITE_OK; - pShmNode = p->pShmNode; +/* +** Make sure all writes to a particular file are committed to disk. +*/ +static int winSync(sqlite3_file *id, int flags){ +#if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || defined(SQLITE_DEBUG) + winFile *pFile = (winFile*)id; + BOOL rc; +#else + UNUSED_PARAMETER(id); +#endif - /* Remove connection p from the set of connections associated - ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); - for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} - *pp = p->pNext; + assert( pFile ); + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); - /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too */ - winShmEnterMutex(); - assert( pShmNode->nRef>0 ); - pShmNode->nRef--; - if( pShmNode->nRef==0 ){ - winShmPurge(pDbFd->pVfs, deleteFlag); + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( (flags&0x0F)==SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; } - winShmLeaveMutex(); + sqlite3_sync_count++; +#endif + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC return SQLITE_OK; +#else + rc = FlushFileBuffers(pFile->h); + SimulateIOError( rc=FALSE ); + if( rc ){ + return SQLITE_OK; + }else{ + pFile->lastErrno = GetLastError(); + return winLogError(SQLITE_IOERR_FSYNC, "winSync", pFile->zPath); + } +#endif } /* -** Change the lock state for a shared-memory segment. +** Determine the current size of a file in bytes */ -static int winShmLock( - sqlite3_file *fd, /* Database file holding the shared memory */ - int ofst, /* First lock to acquire or release */ - int n, /* Number of locks to acquire or release */ - int flags /* What to do with the lock */ -){ - winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ - winShm *p = pDbFd->pShm; /* The shared memory being locked */ - winShm *pX; /* For looping over all siblings */ - winShmNode *pShmNode = p->pShmNode; - int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ - - assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); - assert( n>=1 ); - assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) - || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) - || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) - || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); - assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - - mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - u16 allMask = 0; /* Mask of locks held by siblings */ - - /* See if any siblings hold this same lock */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( pX==p ) continue; - assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); - allMask |= pX->sharedMask; - } +static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ + DWORD upperBits; + DWORD lowerBits; + winFile *pFile = (winFile*)id; + DWORD error; - /* Unlock the system-level locks */ - if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } + assert( id!=0 ); + SimulateIOError(return SQLITE_IOERR_FSTAT); + lowerBits = GetFileSize(pFile->h, &upperBits); + if( (lowerBits == INVALID_FILE_SIZE) + && ((error = GetLastError()) != NO_ERROR) ) + { + pFile->lastErrno = error; + return winLogError(SQLITE_IOERR_FSTAT, "winFileSize", pFile->zPath); + } + *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; + return SQLITE_OK; +} - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - }else if( flags & SQLITE_SHM_SHARED ){ - u16 allShared = 0; /* Union of locks held by connections other than "p" */ +/* +** LOCKFILE_FAIL_IMMEDIATELY is undefined on some Windows systems. +*/ +#ifndef LOCKFILE_FAIL_IMMEDIATELY +# define LOCKFILE_FAIL_IMMEDIATELY 1 +#endif - /* Find out which shared locks are already held by sibling connections. - ** If any sibling already holds an exclusive lock, go ahead and return - ** SQLITE_BUSY. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - allShared |= pX->sharedMask; - } - - /* Get shared locks at the system level, if necessary */ - if( rc==SQLITE_OK ){ - if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } - } - - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - } +/* +** Acquire a reader lock. +** Different API routines are called depending on whether or not this +** is Win95 or WinNT. +*/ +static int getReadLock(winFile *pFile){ + int res; + if( isNT() ){ + OVERLAPPED ovlp; + ovlp.Offset = SHARED_FIRST; + ovlp.OffsetHigh = 0; + ovlp.hEvent = 0; + res = LockFileEx(pFile->h, LOCKFILE_FAIL_IMMEDIATELY, + 0, SHARED_SIZE, 0, &ovlp); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +*/ +#if SQLITE_OS_WINCE==0 }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - } - - /* Get the exclusive locks at the system level. Then if successful - ** also mark the local connection as being locked. - */ - if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); - if( rc==SQLITE_OK ){ - assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; - } - } + int lk; + sqlite3_randomness(sizeof(lk), &lk); + pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); + res = LockFile(pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); +#endif } - sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)GetCurrentProcessId(), p->sharedMask, p->exclMask, - rc ? "failed" : "ok")); - return rc; + if( res == 0 ){ + pFile->lastErrno = GetLastError(); + /* No need to log a failure to lock */ + } + return res; } /* -** Implement a memory barrier or memory fence on shared memory. -** -** All loads and stores begun before the barrier must complete before -** any load or store begun after the barrier. +** Undo a readlock */ -static void winShmBarrier( - sqlite3_file *fd /* Database holding the shared memory */ -){ - UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); - winShmLeaveMutex(); +static int unlockReadLock(winFile *pFile){ + int res; + if( isNT() ){ + res = UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + res = UnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); +#endif + } + if( res==0 && GetLastError()!=ERROR_NOT_LOCKED ){ + pFile->lastErrno = GetLastError(); + winLogError(SQLITE_IOERR_UNLOCK, "unlockReadLock", pFile->zPath); + } + return res; } /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion -** bytes in size. +** Lock the file with the lock specified by parameter locktype - one +** of the following: ** -** If an error occurs, an error code is returned and *pp is set to NULL. +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK ** -** Otherwise, if the isWrite parameter is 0 and the requested shared-memory -** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** isWrite is non-zero and the requested shared-memory region has not yet -** been allocated, it is allocated by this function. +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: ** -** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped -** memory and SQLITE_OK returned. +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. The winUnlock() routine +** erases all locks at once and returns us immediately to locking level 0. +** It is not possible to lower the locking level one step at a time. You +** must go straight to locking level 0. */ -static int winShmMap( - sqlite3_file *fd, /* Handle open on database file */ - int iRegion, /* Region to retrieve */ - int szRegion, /* Size of regions */ - int isWrite, /* True to extend file if necessary */ - void volatile **pp /* OUT: Mapped memory */ -){ - winFile *pDbFd = (winFile*)fd; - winShm *p = pDbFd->pShm; - winShmNode *pShmNode; - int rc = SQLITE_OK; - - if( !p ){ - rc = winOpenSharedMemory(pDbFd); - if( rc!=SQLITE_OK ) return rc; - p = pDbFd->pShm; - } - pShmNode = p->pShmNode; +static int winLock(sqlite3_file *id, int locktype){ + int rc = SQLITE_OK; /* Return code from subroutines */ + int res = 1; /* Result of a windows lock call */ + int newLocktype; /* Set pFile->locktype to this value before exiting */ + int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ + winFile *pFile = (winFile*)id; + DWORD error = NO_ERROR; - sqlite3_mutex_enter(pShmNode->mutex); - assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( id!=0 ); + OSTRACE(("LOCK %d %d was %d(%d)\n", + pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); - if( pShmNode->nRegion<=iRegion ){ - struct ShmRegion *apNew; /* New aRegion[] array */ - int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ - sqlite3_int64 sz; /* Current size of wal-index file */ + /* If there is already a lock of this type or more restrictive on the + ** OsFile, do nothing. Don't use the end_lock: exit path, as + ** sqlite3OsEnterMutex() hasn't been called yet. + */ + if( pFile->locktype>=locktype ){ + return SQLITE_OK; + } - pShmNode->szRegion = szRegion; + /* Make sure the locking sequence is correct + */ + assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); + assert( locktype!=PENDING_LOCK ); + assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); - /* The requested region is not mapped into this processes address space. - ** Check to see if it has been allocated (i.e. if the wal-index file is - ** large enough to contain the requested region). - */ - rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); - if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMSIZE; - goto shmpage_out; + /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of + ** the PENDING_LOCK byte is temporary. + */ + newLocktype = pFile->locktype; + if( (pFile->locktype==NO_LOCK) + || ( (locktype==EXCLUSIVE_LOCK) + && (pFile->locktype==RESERVED_LOCK)) + ){ + int cnt = 3; + while( cnt-->0 && (res = LockFile(pFile->h, PENDING_BYTE, 0, 1, 0))==0 ){ + /* Try 3 times to get the pending lock. The pending lock might be + ** held by another reader process who will release it momentarily. + */ + OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); + Sleep(1); + } + gotPendingLock = res; + if( !res ){ + error = GetLastError(); } + } - if( szhFile, nByte); - if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMSIZE; - goto shmpage_out; - } + /* Acquire a shared lock + */ + if( locktype==SHARED_LOCK && res ){ + assert( pFile->locktype==NO_LOCK ); + res = getReadLock(pFile); + if( res ){ + newLocktype = SHARED_LOCK; + }else{ + error = GetLastError(); } + } - /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc( - pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) - ); - if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; - goto shmpage_out; + /* Acquire a RESERVED lock + */ + if( locktype==RESERVED_LOCK && res ){ + assert( pFile->locktype==SHARED_LOCK ); + res = LockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + if( res ){ + newLocktype = RESERVED_LOCK; + }else{ + error = GetLastError(); } - pShmNode->aRegion = apNew; + } - while( pShmNode->nRegion<=iRegion ){ - HANDLE hMap; /* file-mapping handle */ - void *pMap = 0; /* Mapped memory region */ - - hMap = CreateFileMapping(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL - ); - OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", - (int)GetCurrentProcessId(), pShmNode->nRegion, nByte, - hMap ? "ok" : "failed")); - if( hMap ){ - int iOffset = pShmNode->nRegion*szRegion; - int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; - pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, - 0, iOffset - iOffsetShift, szRegion + iOffsetShift - ); - OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", - (int)GetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, - pMap ? "ok" : "failed")); - } - if( !pMap ){ - pShmNode->lastErrno = GetLastError(); - rc = SQLITE_IOERR; - if( hMap ) CloseHandle(hMap); - goto shmpage_out; - } + /* Acquire a PENDING lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + newLocktype = PENDING_LOCK; + gotPendingLock = 0; + } - pShmNode->aRegion[pShmNode->nRegion].pMap = pMap; - pShmNode->aRegion[pShmNode->nRegion].hMap = hMap; - pShmNode->nRegion++; + /* Acquire an EXCLUSIVE lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + assert( pFile->locktype>=SHARED_LOCK ); + res = unlockReadLock(pFile); + OSTRACE(("unreadlock = %d\n", res)); + res = LockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + if( res ){ + newLocktype = EXCLUSIVE_LOCK; + }else{ + error = GetLastError(); + OSTRACE(("error-code = %d\n", error)); + getReadLock(pFile); } } -shmpage_out: - if( pShmNode->nRegion>iRegion ){ - int iOffset = iRegion*szRegion; - int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; - char *p = (char *)pShmNode->aRegion[iRegion].pMap; - *pp = (void *)&p[iOffsetShift]; + /* If we are holding a PENDING lock that ought to be released, then + ** release it now. + */ + if( gotPendingLock && locktype==SHARED_LOCK ){ + UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); + } + + /* Update the state of the lock has held in the file descriptor then + ** return the appropriate result code. + */ + if( res ){ + rc = SQLITE_OK; }else{ - *pp = 0; + OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, + locktype, newLocktype)); + pFile->lastErrno = error; + rc = SQLITE_BUSY; } - sqlite3_mutex_leave(pShmNode->mutex); + pFile->locktype = (u8)newLocktype; return rc; } -#else -# define winShmMap 0 -# define winShmLock 0 -# define winShmBarrier 0 -# define winShmUnmap 0 -#endif /* #ifndef SQLITE_OMIT_WAL */ - -/* -** Here ends the implementation of all sqlite3_file methods. -** -********************** End sqlite3_file Methods ******************************* -******************************************************************************/ - /* -** This vector defines all the methods that can operate on an -** sqlite3_file for win32. +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, return +** non-zero, otherwise zero. */ -static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ - winClose, /* xClose */ - winRead, /* xRead */ - winWrite, /* xWrite */ - winTruncate, /* xTruncate */ - winSync, /* xSync */ - winFileSize, /* xFileSize */ - winLock, /* xLock */ - winUnlock, /* xUnlock */ - winCheckReservedLock, /* xCheckReservedLock */ - winFileControl, /* xFileControl */ - winSectorSize, /* xSectorSize */ - winDeviceCharacteristics, /* xDeviceCharacteristics */ - winShmMap, /* xShmMap */ - winShmLock, /* xShmLock */ - winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ -}; +static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc; + winFile *pFile = (winFile*)id; -/**************************************************************************** -**************************** sqlite3_vfs methods **************************** -** -** This division contains the implementation of methods on the -** sqlite3_vfs object. -*/ + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); -/* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from malloc and must be freed by the calling -** function. -*/ -static void *convertUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( isNT() ){ - zConverted = utf8ToUnicode(zFilename); -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -*/ -#if SQLITE_OS_WINCE==0 + assert( id!=0 ); + if( pFile->locktype>=RESERVED_LOCK ){ + rc = 1; + OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); }else{ - zConverted = utf8ToMbcs(zFilename); -#endif + rc = LockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + if( rc ){ + UnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); + } + rc = !rc; + OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); } - /* caller will handle out of memory */ - return zConverted; + *pResOut = rc; + return SQLITE_OK; } /* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at pVfs->mxPathname characters. -*/ -static int getTempname(int nBuf, char *zBuf){ - static char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - size_t i, j; - char zTempPath[MAX_PATH+1]; - - /* It's odd to simulate an io-error here, but really this is just - ** using the io-error infrastructure to test that SQLite handles this - ** function failing. - */ - SimulateIOError( return SQLITE_IOERR ); - - if( sqlite3_temp_directory ){ - sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", sqlite3_temp_directory); - }else if( isNT() ){ - char *zMulti; - WCHAR zWidePath[MAX_PATH]; - GetTempPathW(MAX_PATH-30, zWidePath); - zMulti = unicodeToUtf8(zWidePath); - if( zMulti ){ - sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); - free(zMulti); - }else{ - return SQLITE_NOMEM; - } -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. +** Lower the locking level on file descriptor id to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** It is not possible for this routine to fail if the second argument +** is NO_LOCK. If the second argument is SHARED_LOCK then this routine +** might return SQLITE_IOERR; */ -#if SQLITE_OS_WINCE==0 - }else{ - char *zUtf8; - char zMbcsPath[MAX_PATH]; - GetTempPathA(MAX_PATH-30, zMbcsPath); - zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); - if( zUtf8 ){ - sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); - free(zUtf8); - }else{ - return SQLITE_NOMEM; +static int winUnlock(sqlite3_file *id, int locktype){ + int type; + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + assert( pFile!=0 ); + assert( locktype<=SHARED_LOCK ); + OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, + pFile->locktype, pFile->sharedLockByte)); + type = pFile->locktype; + if( type>=EXCLUSIVE_LOCK ){ + UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + if( locktype==SHARED_LOCK && !getReadLock(pFile) ){ + /* This should never happen. We should always be able to + ** reacquire the read lock */ + rc = winLogError(SQLITE_IOERR_UNLOCK, "winUnlock", pFile->zPath); } -#endif } - - /* Check that the output buffer is large enough for the temporary file - ** name. If it is not, return SQLITE_ERROR. - */ - if( (sqlite3Strlen30(zTempPath) + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 17) >= nBuf ){ - return SQLITE_ERROR; + if( type>=RESERVED_LOCK ){ + UnlockFile(pFile->h, RESERVED_BYTE, 0, 1, 0); } - - for(i=sqlite3Strlen30(zTempPath); i>0 && zTempPath[i-1]=='\\'; i--){} - zTempPath[i] = 0; - - sqlite3_snprintf(nBuf-17, zBuf, - "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath); - j = sqlite3Strlen30(zBuf); - sqlite3_randomness(15, &zBuf[j]); - for(i=0; i<15; i++, j++){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + if( locktype==NO_LOCK && type>=SHARED_LOCK ){ + unlockReadLock(pFile); } - zBuf[j] = 0; - - OSTRACE(("TEMP FILENAME: %s\n", zBuf)); - return SQLITE_OK; + if( type>=PENDING_LOCK ){ + UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); + } + pFile->locktype = (u8)locktype; + return rc; } /* -** The return value of getLastErrorMsg -** is zero if the error message fits in the buffer, or non-zero -** otherwise (if the message was truncated). +** Control and query of the open file handle. */ -static int getLastErrorMsg(int nBuf, char *zBuf){ - /* FormatMessage returns 0 on failure. Otherwise it - ** returns the number of TCHARs written to the output - ** buffer, excluding the terminating null char. - */ - DWORD error = GetLastError(); - DWORD dwLen = 0; - char *zOut = 0; - - if( isNT() ){ - WCHAR *zTempWide = NULL; - dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPWSTR) &zTempWide, - 0, - 0); - if( dwLen > 0 ){ - /* allocate a buffer and convert to UTF8 */ - zOut = unicodeToUtf8(zTempWide); - /* free the system buffer allocated by FormatMessage */ - LocalFree(zTempWide); +static int winFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = ((winFile*)id)->locktype; + return SQLITE_OK; } -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 - }else{ - char *zTemp = NULL; - dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPSTR) &zTemp, - 0, - 0); - if( dwLen > 0 ){ - /* allocate a buffer and convert to UTF8 */ - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); - /* free the system buffer allocated by FormatMessage */ - LocalFree(zTemp); + case SQLITE_LAST_ERRNO: { + *(int*)pArg = (int)((winFile*)id)->lastErrno; + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((winFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + sqlite3_int64 sz = *(sqlite3_int64*)pArg; + SimulateIOErrorBenign(1); + winTruncate(id, sz); + SimulateIOErrorBenign(0); + return SQLITE_OK; + } + case SQLITE_FCNTL_SYNC_OMITTED: { + return SQLITE_OK; } -#endif - } - if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); - }else{ - /* copy a maximum of nBuf chars to output buffer */ - sqlite3_snprintf(nBuf, zBuf, "%s", zOut); - /* free the UTF8 buffer */ - free(zOut); } - return 0; + return SQLITE_NOTFOUND; } /* -** Open a file. +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. */ -static int winOpen( - sqlite3_vfs *pVfs, /* Not used */ - const char *zName, /* Name of the file (UTF-8) */ - sqlite3_file *id, /* Write the SQLite file handle here */ - int flags, /* Open mode flags */ - int *pOutFlags /* Status return flags */ -){ - HANDLE h; - DWORD dwDesiredAccess; - DWORD dwShareMode; - DWORD dwCreationDisposition; - DWORD dwFlagsAndAttributes = 0; -#if SQLITE_OS_WINCE - int isTemp = 0; -#endif - winFile *pFile = (winFile*)id; - void *zConverted; /* Filename in OS encoding */ - const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ +static int winSectorSize(sqlite3_file *id){ + assert( id!=0 ); + return (int)(((winFile*)id)->sectorSize); +} - /* If argument zPath is a NULL pointer, this function is required to open - ** a temporary file. Use this buffer to store the file name in. - */ - char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ +/* +** Return a vector of device characteristics. +*/ +static int winDeviceCharacteristics(sqlite3_file *id){ + UNUSED_PARAMETER(id); + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; +} - int rc = SQLITE_OK; /* Function Return Code */ -#if !defined(NDEBUG) || SQLITE_OS_WINCE - int eType = flags&0xFFFFFF00; /* Type of file to open */ -#endif +#ifndef SQLITE_OMIT_WAL - int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); - int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); - int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG - int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif - int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +SYSTEM_INFO winSysInfo; -#ifndef NDEBUG - int isOpenJournal = (isCreate && ( - eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_MAIN_JOURNAL - || eType==SQLITE_OPEN_WAL - )); +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the winLockInfo objects used by +** this file, all of which may be shared by multiple threads. +** +** Function winShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** winShmEnterMutex() +** assert( winShmMutexHeld() ); +** winShmLeaveMutex() +*/ +static void winShmEnterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +static void winShmLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +#ifdef SQLITE_DEBUG +static int winShmMutexHeld(void) { + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} #endif - /* Check the following statements are true: - ** - ** (a) Exactly one of the READWRITE and READONLY flags must be set, and - ** (b) if CREATE is set, then READWRITE must also be set, and - ** (c) if EXCLUSIVE is set, then CREATE must also be set. - ** (d) if DELETEONCLOSE is set, then CREATE must also be set. - */ - assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); - assert(isCreate==0 || isReadWrite); - assert(isExclusive==0 || isCreate); - assert(isDelete==0 || isCreate); +/* +** Object used to represent a single file opened and mmapped to provide +** shared memory. When multiple threads all reference the same +** log-summary, each thread has its own winFile object, but they all +** point to a single instance of this object. In other words, each +** log-summary is opened only once per process. +** +** winShmMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** pNext +** +** The following fields are read-only after the object is created: +** +** fid +** zFilename +** +** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** winShmMutexHeld() is true when reading or writing any other field +** in this structure. +** +*/ +struct winShmNode { + sqlite3_mutex *mutex; /* Mutex to access this object */ + char *zFilename; /* Name of the file */ + winFile hFile; /* File handle from winOpen */ - /* The main DB, main journal, WAL file and master journal are never - ** automatically deleted. Nor are they ever temporary files. */ - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + int szRegion; /* Size of shared-memory regions */ + int nRegion; /* Size of array apRegion */ + struct ShmRegion { + HANDLE hMap; /* File handle from CreateFileMapping */ + void *pMap; + } *aRegion; + DWORD lastErrno; /* The Windows errno from the last I/O error */ - /* Assert that the upper layer has set one of the "file-type" flags. */ - assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB - || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL - ); + int nRef; /* Number of winShm objects pointing to this */ + winShm *pFirst; /* All winShm objects pointing to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ +#ifdef SQLITE_DEBUG + u8 nextShmId; /* Next available winShm.id value */ +#endif +}; - assert( id!=0 ); - UNUSED_PARAMETER(pVfs); +/* +** A global array of all winShmNode objects. +** +** The winShmMutexHeld() must be true while reading or writing this list. +*/ +static winShmNode *winShmNodeList = 0; - pFile->h = INVALID_HANDLE_VALUE; +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** winShm.pShmNode +** winShm.id +** +** All other fields are read/write. The winShm.pShmNode->mutex must be held +** while accessing any read/write fields. +*/ +struct winShm { + winShmNode *pShmNode; /* The underlying winShmNode object */ + winShm *pNext; /* Next winShm with the same winShmNode */ + u8 hasMutex; /* True if holding the winShmNode mutex */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_DEBUG + u8 id; /* Id of this connection with its winShmNode */ +#endif +}; - /* If the second argument to this function is NULL, generate a - ** temporary file name to use - */ - if( !zUtf8Name ){ - assert(isDelete && !isOpenJournal); - rc = getTempname(MAX_PATH+1, zTmpname); - if( rc!=SQLITE_OK ){ - return rc; - } - zUtf8Name = zTmpname; - } +/* +** Constants used for locking +*/ +#define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ - /* Convert the filename to the system encoding. */ - zConverted = convertUtf8Filename(zUtf8Name); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } +/* +** Apply advisory locks for all n bytes beginning at ofst. +*/ +#define _SHM_UNLCK 1 +#define _SHM_RDLCK 2 +#define _SHM_WRLCK 3 +static int winShmSystemLock( + winShmNode *pFile, /* Apply locks to this open shared-memory segment */ + int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ + int ofst, /* Offset to first byte to be locked/unlocked */ + int nByte /* Number of bytes to lock or unlock */ +){ + OVERLAPPED ovlp; + DWORD dwFlags; + int rc = 0; /* Result code form Lock/UnlockFileEx() */ - if( isReadWrite ){ - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - }else{ - dwDesiredAccess = GENERIC_READ; - } + /* Access to the winShmNode object is serialized by the caller */ + assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); - /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is - ** created. SQLite doesn't use it to indicate "exclusive access" - ** as it is usually understood. - */ - if( isExclusive ){ - /* Creates a new file, only if it does not already exist. */ - /* If the file exists, it fails. */ - dwCreationDisposition = CREATE_NEW; - }else if( isCreate ){ - /* Open existing file, or create if it doesn't exist */ - dwCreationDisposition = OPEN_ALWAYS; - }else{ - /* Opens a file, only if it exists. */ - dwCreationDisposition = OPEN_EXISTING; - } + /* Initialize the locking parameters */ + dwFlags = LOCKFILE_FAIL_IMMEDIATELY; + if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; - dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = ofst; - if( isDelete ){ -#if SQLITE_OS_WINCE - dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; - isTemp = 1; -#else - dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY - | FILE_ATTRIBUTE_HIDDEN - | FILE_FLAG_DELETE_ON_CLOSE; -#endif + /* Release/Acquire the system-level lock */ + if( lockType==_SHM_UNLCK ){ + rc = UnlockFileEx(pFile->hFile.h, 0, nByte, 0, &ovlp); }else{ - dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + rc = LockFileEx(pFile->hFile.h, dwFlags, 0, nByte, 0, &ovlp); } - /* Reports from the internet are that performance is always - ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ -#if SQLITE_OS_WINCE - dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; -#endif - - if( isNT() ){ - h = CreateFileW((WCHAR*)zConverted, - dwDesiredAccess, - dwShareMode, - NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL - ); -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 + + if( rc!= 0 ){ + rc = SQLITE_OK; }else{ - h = CreateFileA((char*)zConverted, - dwDesiredAccess, - dwShareMode, - NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL - ); -#endif + pFile->lastErrno = GetLastError(); + rc = SQLITE_BUSY; } - OSTRACE(("OPEN %d %s 0x%lx %s\n", - h, zName, dwDesiredAccess, - h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", + pFile->hFile.h, + rc==SQLITE_OK ? "ok" : "failed", + lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", + pFile->lastErrno)); - if( h==INVALID_HANDLE_VALUE ){ - pFile->lastErrno = GetLastError(); - free(zConverted); - if( isReadWrite ){ - return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); - }else{ - return SQLITE_CANTOPEN_BKPT; - } - } + return rc; +} - if( pOutFlags ){ - if( isReadWrite ){ - *pOutFlags = SQLITE_OPEN_READWRITE; +/* Forward references to VFS methods */ +static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); +static int winDelete(sqlite3_vfs *,const char*,int); + +/* +** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. +*/ +static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ + winShmNode **pp; + winShmNode *p; + BOOL bRc; + assert( winShmMutexHeld() ); + pp = &winShmNodeList; + while( (p = *pp)!=0 ){ + if( p->nRef==0 ){ + int i; + if( p->mutex ) sqlite3_mutex_free(p->mutex); + for(i=0; inRegion; i++){ + bRc = UnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); + bRc = CloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); + } + if( p->hFile.h != INVALID_HANDLE_VALUE ){ + SimulateIOErrorBenign(1); + winClose((sqlite3_file *)&p->hFile); + SimulateIOErrorBenign(0); + } + if( deleteFlag ){ + SimulateIOErrorBenign(1); + winDelete(pVfs, p->zFilename, 0); + SimulateIOErrorBenign(0); + } + *pp = p->pNext; + sqlite3_free(p->aRegion); + sqlite3_free(p); }else{ - *pOutFlags = SQLITE_OPEN_READONLY; + pp = &p->pNext; } } - - memset(pFile, 0, sizeof(*pFile)); - pFile->pMethod = &winIoMethod; - pFile->h = h; - pFile->lastErrno = NO_ERROR; - pFile->pVfs = pVfs; - pFile->pShm = 0; - pFile->zPath = zName; - pFile->sectorSize = getSectorSize(pVfs, zUtf8Name); - -#if SQLITE_OS_WINCE - if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && !winceCreateLock(zName, pFile) - ){ - CloseHandle(h); - free(zConverted); - return SQLITE_CANTOPEN_BKPT; - } - if( isTemp ){ - pFile->zDeleteOnClose = zConverted; - }else -#endif - { - free(zConverted); - } - - OpenCounter(+1); - return rc; } /* -** Delete the named file. +** Open the shared-memory area associated with database file pDbFd. ** -** Note that windows does not allow a file to be deleted if some other -** process has it open. Sometimes a virus scanner or indexing program -** will open a journal file shortly after it is created in order to do -** whatever it does. While this other process is holding the -** file open, we will be unable to delete it. To work around this -** problem, we delay 100 milliseconds and try to delete again. Up -** to MX_DELETION_ATTEMPTs deletion attempts are run before giving -** up and returning an error. +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. */ -#define MX_DELETION_ATTEMPTS 5 -static int winDelete( - sqlite3_vfs *pVfs, /* Not used on win32 */ - const char *zFilename, /* Name of file to delete */ - int syncDir /* Not used on win32 */ -){ - int cnt = 0; - DWORD rc; - DWORD error = 0; - void *zConverted; - UNUSED_PARAMETER(pVfs); - UNUSED_PARAMETER(syncDir); +static int winOpenSharedMemory(winFile *pDbFd){ + struct winShm *p; /* The connection to be opened */ + struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc; /* Result code */ + struct winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ - SimulateIOError(return SQLITE_IOERR_DELETE); - zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ + assert( pDbFd->pShm==0 ); /* Not previously opened */ + + /* Allocate space for the new sqlite3_shm object. Also speculatively + ** allocate space for a new winShmNode and filename. + */ + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + nName = sqlite3Strlen30(pDbFd->zPath); + pNew = sqlite3_malloc( sizeof(*pShmNode) + nName + 15 ); + if( pNew==0 ){ + sqlite3_free(p); return SQLITE_NOMEM; } - if( isNT() ){ - do{ - DeleteFileW(zConverted); - }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 - }else{ - do{ - DeleteFileA(zConverted); - }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); -#endif + memset(pNew, 0, sizeof(*pNew)); + pNew->zFilename = (char*)&pNew[1]; + sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + + /* Look to see if there is an existing winShmNode that can be used. + ** If no matching winShmNode currently exists, create a new one. + */ + winShmEnterMutex(); + for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ + /* TBD need to come up with better match here. Perhaps + ** use FILE_ID_BOTH_DIR_INFO Structure. + */ + if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; } - free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, - ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ? - "ok" : "failed" )); - - return ( (rc == INVALID_FILE_ATTRIBUTES) - && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : SQLITE_IOERR_DELETE; -} + if( pShmNode ){ + sqlite3_free(pNew); + }else{ + pShmNode = pNew; + pNew = 0; + ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; -/* -** Check the existance and status of a file. -*/ -static int winAccess( - sqlite3_vfs *pVfs, /* Not used on win32 */ - const char *zFilename, /* Name of file to check */ - int flags, /* Type of test to make on this file */ - int *pResOut /* OUT: Result */ -){ - DWORD attr; - int rc = 0; - void *zConverted; - UNUSED_PARAMETER(pVfs); + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } - SimulateIOError( return SQLITE_IOERR_ACCESS; ); - zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - if( isNT() ){ - WIN32_FILE_ATTRIBUTE_DATA sAttrData; - memset(&sAttrData, 0, sizeof(sAttrData)); - if( GetFileAttributesExW((WCHAR*)zConverted, - GetFileExInfoStandard, - &sAttrData) ){ - /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file - ** as if it does not exist. - */ - if( flags==SQLITE_ACCESS_EXISTS - && sAttrData.nFileSizeHigh==0 - && sAttrData.nFileSizeLow==0 ){ - attr = INVALID_FILE_ATTRIBUTES; - }else{ - attr = sAttrData.dwFileAttributes; - } - }else{ - if( GetLastError()!=ERROR_FILE_NOT_FOUND ){ - free(zConverted); - return SQLITE_IOERR_ACCESS; - }else{ - attr = INVALID_FILE_ATTRIBUTES; + rc = winOpen(pDbFd->pVfs, + pShmNode->zFilename, /* Name of the file (UTF-8) */ + (sqlite3_file*)&pShmNode->hFile, /* File handle here */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + 0); + if( SQLITE_OK!=rc ){ + rc = SQLITE_CANTOPEN_BKPT; + goto shm_open_err; + } + + /* Check to see if another process is holding the dead-man switch. + ** If not, truncate the file to zero length. + */ + if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ + rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMOPEN, "winOpenShm", pDbFd->zPath); } } -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 - }else{ - attr = GetFileAttributesA((char*)zConverted); -#endif - } - free(zConverted); - switch( flags ){ - case SQLITE_ACCESS_READ: - case SQLITE_ACCESS_EXISTS: - rc = attr!=INVALID_FILE_ATTRIBUTES; - break; - case SQLITE_ACCESS_READWRITE: - rc = (attr & FILE_ATTRIBUTE_READONLY)==0; - break; - default: - assert(!"Invalid flags argument"); + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); + } + if( rc ) goto shm_open_err; } - *pResOut = rc; + + /* Make the new connection a child of the winShmNode */ + p->pShmNode = pShmNode; +#ifdef SQLITE_DEBUG + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + winShmLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the winShmEnterMutex() mutex and the pointer from the + ** new (struct winShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); return SQLITE_OK; -} + /* Jump here on any error */ +shm_open_err: + winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ + sqlite3_free(p); + sqlite3_free(pNew); + winShmLeaveMutex(); + return rc; +} /* -** Turn a relative pathname into a full pathname. Write the full -** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname -** bytes in size. +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. */ -static int winFullPathname( - sqlite3_vfs *pVfs, /* Pointer to vfs object */ - const char *zRelative, /* Possibly relative input path */ - int nFull, /* Size of output buffer in bytes */ - char *zFull /* Output buffer */ +static int winShmUnmap( + sqlite3_file *fd, /* Database holding shared memory */ + int deleteFlag /* Delete after closing if true */ ){ - -#if defined(__CYGWIN__) - SimulateIOError( return SQLITE_ERROR ); - UNUSED_PARAMETER(nFull); - cygwin_conv_to_full_win32_path(zRelative, zFull); - return SQLITE_OK; -#endif + winFile *pDbFd; /* Database holding shared-memory */ + winShm *p; /* The connection to be closed */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + winShm **pp; /* For looping over sibling connections */ -#if SQLITE_OS_WINCE - SimulateIOError( return SQLITE_ERROR ); - UNUSED_PARAMETER(nFull); - /* WinCE has no concept of a relative pathname, or so I am told. */ - sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zRelative); - return SQLITE_OK; -#endif + pDbFd = (winFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; -#if !SQLITE_OS_WINCE && !defined(__CYGWIN__) - int nByte; - void *zConverted; - char *zOut; + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; - /* It's odd to simulate an io-error here, but really this is just - ** using the io-error infrastructure to test that SQLite handles this - ** function failing. This function could fail if, for example, the - ** current working directory has been unlinked. - */ - SimulateIOError( return SQLITE_ERROR ); - UNUSED_PARAMETER(nFull); - zConverted = convertUtf8Filename(zRelative); - if( isNT() ){ - WCHAR *zTemp; - nByte = GetFullPathNameW((WCHAR*)zConverted, 0, 0, 0) + 3; - zTemp = malloc( nByte*sizeof(zTemp[0]) ); - if( zTemp==0 ){ - free(zConverted); - return SQLITE_NOMEM; - } - GetFullPathNameW((WCHAR*)zConverted, nByte, zTemp, 0); - free(zConverted); - zOut = unicodeToUtf8(zTemp); - free(zTemp); -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 - }else{ - char *zTemp; - nByte = GetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; - zTemp = malloc( nByte*sizeof(zTemp[0]) ); - if( zTemp==0 ){ - free(zConverted); - return SQLITE_NOMEM; - } - GetFullPathNameA((char*)zConverted, nByte, zTemp, 0); - free(zConverted); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); - free(zTemp); -#endif - } - if( zOut ){ - sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zOut); - free(zOut); - return SQLITE_OK; - }else{ - return SQLITE_NOMEM; + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->mutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + winShmEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + winShmPurge(pDbFd->pVfs, deleteFlag); } -#endif + winShmLeaveMutex(); + + return SQLITE_OK; } /* -** Get the sector size of the device used to store -** file. +** Change the lock state for a shared-memory segment. */ -static int getSectorSize( - sqlite3_vfs *pVfs, - const char *zRelative /* UTF-8 file name */ +static int winShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ ){ - DWORD bytesPerSector = SQLITE_DEFAULT_SECTOR_SIZE; - /* GetDiskFreeSpace is not supported under WINCE */ -#if SQLITE_OS_WINCE - UNUSED_PARAMETER(pVfs); - UNUSED_PARAMETER(zRelative); -#else - char zFullpath[MAX_PATH+1]; - int rc; - DWORD dwRet = 0; - DWORD dwDummy; + winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ + winShm *p = pDbFd->pShm; /* The shared memory being locked */ + winShm *pX; /* For looping over all siblings */ + winShmNode *pShmNode = p->pShmNode; + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ - /* - ** We need to get the full path name of the file - ** to get the drive letter to look up the sector - ** size. - */ - SimulateIOErrorBenign(1); - rc = winFullPathname(pVfs, zRelative, MAX_PATH, zFullpath); - SimulateIOErrorBenign(0); - if( rc == SQLITE_OK ) - { - void *zConverted = convertUtf8Filename(zFullpath); - if( zConverted ){ - if( isNT() ){ - /* trim path to just drive reference */ - WCHAR *p = zConverted; - for(;*p;p++){ - if( *p == '\\' ){ - *p = '\0'; - break; - } - } - dwRet = GetDiskFreeSpaceW((WCHAR*)zConverted, - &dwDummy, - &bytesPerSector, - &dwDummy, - &dwDummy); + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + + mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ + + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); }else{ - /* trim path to just drive reference */ - char *p = (char *)zConverted; - for(;*p;p++){ - if( *p == '\\' ){ - *p = '\0'; - break; - } - } - dwRet = GetDiskFreeSpaceA((char*)zConverted, - &dwDummy, - &bytesPerSector, - &dwDummy, - &dwDummy); + rc = SQLITE_OK; } - free(zConverted); } - if( !dwRet ){ - bytesPerSector = SQLITE_DEFAULT_SECTOR_SIZE; + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } } } -#endif - return (int) bytesPerSector; + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", + p->id, (int)GetCurrentProcessId(), p->sharedMask, p->exclMask, + rc ? "failed" : "ok")); + return rc; } -#ifndef SQLITE_OMIT_LOAD_EXTENSION -/* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ /* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ -static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ - HANDLE h; - void *zConverted = convertUtf8Filename(zFilename); - UNUSED_PARAMETER(pVfs); - if( zConverted==0 ){ - return 0; - } - if( isNT() ){ - h = LoadLibraryW((WCHAR*)zConverted); -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. */ -#if SQLITE_OS_WINCE==0 - }else{ - h = LoadLibraryA((char*)zConverted); -#endif - } - free(zConverted); - return (void*)h; -} -static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ - UNUSED_PARAMETER(pVfs); - getLastErrorMsg(nBuf, zBufOut); -} -void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ - UNUSED_PARAMETER(pVfs); -#if SQLITE_OS_WINCE - /* The GetProcAddressA() routine is only available on wince. */ - return (void(*)(void))GetProcAddressA((HANDLE)pHandle, zSymbol); -#else - /* All other windows platforms expect GetProcAddress() to take - ** an Ansi string regardless of the _UNICODE setting */ - return (void(*)(void))GetProcAddress((HANDLE)pHandle, zSymbol); -#endif -} -void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ - UNUSED_PARAMETER(pVfs); - FreeLibrary((HANDLE)pHandle); +static void winShmBarrier( + sqlite3_file *fd /* Database holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + /* MemoryBarrier(); // does not work -- do not know why not */ + winShmEnterMutex(); + winShmLeaveMutex(); } -#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ - #define winDlOpen 0 - #define winDlError 0 - #define winDlSym 0 - #define winDlClose 0 -#endif - /* -** Write up to nBuf bytes of randomness into zBuf. +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the isWrite parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. */ -static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - int n = 0; - UNUSED_PARAMETER(pVfs); -#if defined(SQLITE_TEST) - n = nBuf; - memset(zBuf, 0, nBuf); -#else - if( sizeof(SYSTEMTIME)<=nBuf-n ){ - SYSTEMTIME x; - GetSystemTime(&x); - memcpy(&zBuf[n], &x, sizeof(x)); - n += sizeof(x); - } - if( sizeof(DWORD)<=nBuf-n ){ - DWORD pid = GetCurrentProcessId(); - memcpy(&zBuf[n], &pid, sizeof(pid)); - n += sizeof(pid); +static int winShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + winFile *pDbFd = (winFile*)fd; + winShm *p = pDbFd->pShm; + winShmNode *pShmNode; + int rc = SQLITE_OK; + + if( !p ){ + rc = winOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + p = pDbFd->pShm; } - if( sizeof(DWORD)<=nBuf-n ){ - DWORD cnt = GetTickCount(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + pShmNode = p->pShmNode; + + sqlite3_mutex_enter(pShmNode->mutex); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + + if( pShmNode->nRegion<=iRegion ){ + struct ShmRegion *apNew; /* New aRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + sqlite3_int64 sz; /* Current size of wal-index file */ + + pShmNode->szRegion = szRegion; + + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap1", pDbFd->zPath); + goto shmpage_out; + } + + if( szhFile, nByte); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap2", pDbFd->zPath); + goto shmpage_out; + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (struct ShmRegion *)sqlite3_realloc( + pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; + } + pShmNode->aRegion = apNew; + + while( pShmNode->nRegion<=iRegion ){ + HANDLE hMap; /* file-mapping handle */ + void *pMap = 0; /* Mapped memory region */ + + hMap = CreateFileMapping(pShmNode->hFile.h, + NULL, PAGE_READWRITE, 0, nByte, NULL + ); + OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); + if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; + pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + 0, iOffset - iOffsetShift, szRegion + iOffsetShift + ); + OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, + pMap ? "ok" : "failed")); + } + if( !pMap ){ + pShmNode->lastErrno = GetLastError(); + rc = winLogError(SQLITE_IOERR_SHMMAP, "winShmMap3", pDbFd->zPath); + if( hMap ) CloseHandle(hMap); + goto shmpage_out; + } + + pShmNode->aRegion[pShmNode->nRegion].pMap = pMap; + pShmNode->aRegion[pShmNode->nRegion].hMap = hMap; + pShmNode->nRegion++; + } } - if( sizeof(LARGE_INTEGER)<=nBuf-n ){ - LARGE_INTEGER i; - QueryPerformanceCounter(&i); - memcpy(&zBuf[n], &i, sizeof(i)); - n += sizeof(i); + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; + char *p = (char *)pShmNode->aRegion[iRegion].pMap; + *pp = (void *)&p[iOffsetShift]; + }else{ + *pp = 0; } -#endif - return n; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; } +#else +# define winShmMap 0 +# define winShmLock 0 +# define winShmBarrier 0 +# define winShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ /* -** Sleep for a little while. Return the amount of time slept. +** This vector defines all the methods that can operate on an +** sqlite3_file for win32. +*/ +static const sqlite3_io_methods winIoMethod = { + 2, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winLock, /* xLock */ + winUnlock, /* xUnlock */ + winCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap /* xShmUnmap */ +}; + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. */ -static int winSleep(sqlite3_vfs *pVfs, int microsec){ - Sleep((microsec+999)/1000); - UNUSED_PARAMETER(pVfs); - return ((microsec+999)/1000)*1000; -} /* -** The following variable, if set to a non-zero value, is interpreted as -** the number of seconds since 1970 and is used to set the result of -** sqlite3OsCurrentTime() during testing. +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +static void *convertUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( isNT() ){ + zConverted = utf8ToUnicode(zFilename); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); #endif + } + /* caller will handle out of memory */ + return zConverted; +} /* -** Find the current time (in Universal Coordinated Time). Write into *piNow -** the current time and date as a Julian Day number times 86_400_000. In -** other words, write into *piNow the number of milliseconds since the Julian -** epoch of noon in Greenwich on November 24, 4714 B.C according to the -** proleptic Gregorian calendar. -** -** On success, return 0. Return 1 if the time and date cannot be found. +** Create a temporary file name in zBuf. zBuf must be big enough to +** hold at pVfs->mxPathname characters. */ -static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ - /* FILETIME structure is a 64-bit value representing the number of - 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). +static int getTempname(int nBuf, char *zBuf){ + static char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + size_t i, j; + char zTempPath[MAX_PATH+1]; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. */ - FILETIME ft; - static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; -#ifdef SQLITE_TEST - static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + SimulateIOError( return SQLITE_IOERR ); + + if( sqlite3_temp_directory ){ + sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", sqlite3_temp_directory); + }else if( isNT() ){ + char *zMulti; + WCHAR zWidePath[MAX_PATH]; + GetTempPathW(MAX_PATH-30, zWidePath); + zMulti = unicodeToUtf8(zWidePath); + if( zMulti ){ + sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); + free(zMulti); + }else{ + return SQLITE_NOMEM; + } +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + char *zUtf8; + char zMbcsPath[MAX_PATH]; + GetTempPathA(MAX_PATH-30, zMbcsPath); + zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); + if( zUtf8 ){ + sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); + free(zUtf8); + }else{ + return SQLITE_NOMEM; + } #endif - /* 2^32 - to avoid use of LL and warnings in gcc */ - static const sqlite3_int64 max32BitValue = - (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; + } -#if SQLITE_OS_WINCE - SYSTEMTIME time; - GetSystemTime(&time); - /* if SystemTimeToFileTime() fails, it returns zero. */ - if (!SystemTimeToFileTime(&time,&ft)){ - return 1; + /* Check that the output buffer is large enough for the temporary file + ** name. If it is not, return SQLITE_ERROR. + */ + if( (sqlite3Strlen30(zTempPath) + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 17) >= nBuf ){ + return SQLITE_ERROR; } -#else - GetSystemTimeAsFileTime( &ft ); -#endif - *piNow = winFiletimeEpoch + - ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + - (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; + for(i=sqlite3Strlen30(zTempPath); i>0 && zTempPath[i-1]=='\\'; i--){} + zTempPath[i] = 0; -#ifdef SQLITE_TEST - if( sqlite3_current_time ){ - *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + sqlite3_snprintf(nBuf-17, zBuf, + "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath); + j = sqlite3Strlen30(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } -#endif - UNUSED_PARAMETER(pVfs); - return 0; -} + zBuf[j] = 0; -/* -** Find the current time (in Universal Coordinated Time). Write the -** current time and date as a Julian Day number into *prNow and -** return 0. Return 1 if the time and date cannot be found. -*/ -int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ - int rc; - sqlite3_int64 i; - rc = winCurrentTimeInt64(pVfs, &i); - if( !rc ){ - *prNow = i/86400000.0; - } - return rc; + OSTRACE(("TEMP FILENAME: %s\n", zBuf)); + return SQLITE_OK; } /* -** The idea is that this function works like a combination of -** GetLastError() and FormatMessage() on windows (or errno and -** strerror_r() on unix). After an error is returned by an OS -** function, SQLite calls this function with zBuf pointing to -** a buffer of nBuf bytes. The OS layer should populate the -** buffer with a nul-terminated UTF-8 encoded error message -** describing the last IO error to have occurred within the calling -** thread. -** -** If the error message is too large for the supplied buffer, -** it should be truncated. The return value of xGetLastError -** is zero if the error message fits in the buffer, or non-zero -** otherwise (if the message was truncated). If non-zero is returned, -** then it is not necessary to include the nul-terminator character -** in the output buffer. -** -** Not supplying an error message will have no adverse effect -** on SQLite. It is fine to have an implementation that never -** returns an error message: -** -** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ -** assert(zBuf[0]=='\0'); -** return 0; -** } -** -** However if an error message is supplied, it will be incorporated -** by sqlite into the error message available to the user using -** sqlite3_errmsg(), possibly making IO errors easier to debug. +** Open a file. */ -static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - UNUSED_PARAMETER(pVfs); - return getLastErrorMsg(nBuf, zBuf); -} - +static int winOpen( + sqlite3_vfs *pVfs, /* Not used */ + const char *zName, /* Name of the file (UTF-8) */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HANDLE h; + DWORD dwDesiredAccess; + DWORD dwShareMode; + DWORD dwCreationDisposition; + DWORD dwFlagsAndAttributes = 0; +#if SQLITE_OS_WINCE + int isTemp = 0; +#endif + winFile *pFile = (winFile*)id; + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ -/* -** Initialize and deinitialize the operating system interface. -*/ -SQLITE_API int sqlite3_os_init(void){ - static sqlite3_vfs winVfs = { - 2, /* iVersion */ - sizeof(winFile), /* szOsFile */ - MAX_PATH, /* mxPathname */ - 0, /* pNext */ - "win32", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - }; + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif -#ifndef SQLITE_OMIT_WAL - /* get memory map allocation granularity */ - memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); - GetSystemInfo(&winSysInfo); - assert(winSysInfo.dwAllocationGranularity > 0); + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); +#ifndef NDEBUG + int isReadonly = (flags & SQLITE_OPEN_READONLY); #endif + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); - sqlite3_vfs_register(&winVfs, 1); - return SQLITE_OK; -} -SQLITE_API int sqlite3_os_end(void){ - return SQLITE_OK; -} +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif -#endif /* SQLITE_OS_WIN */ + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); -/************** End of os_win.c **********************************************/ -/************** Begin file bitvec.c ******************************************/ -/* -** 2008 February 16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file implements an object that represents a fixed-length -** bitmap. Bits are numbered starting with 1. -** -** A bitmap is used to record which pages of a database file have been -** journalled during a transaction, or which pages have the "dont-write" -** property. Usually only a few pages are meet either condition. -** So the bitmap is usually sparse and has low cardinality. -** But sometimes (for example when during a DROP of a large table) most -** or all of the pages in a database can get journalled. In those cases, -** the bitmap becomes dense with high cardinality. The algorithm needs -** to handle both cases well. -** -** The size of the bitmap is fixed when the object is created. -** -** All bits are clear when the bitmap is created. Individual bits -** may be set or cleared one at a time. -** -** Test operations are about 100 times more common that set operations. -** Clear operations are exceedingly rare. There are usually between -** 5 and 500 set operations per Bitvec object, though the number of sets can -** sometimes grow into tens of thousands or larger. The size of the -** Bitvec object is the number of pages in the database file at the -** start of a transaction, and is thus usually less than a few thousand, -** but can be as large as 2 billion for a really big database. -*/ + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); -/* Size of the Bitvec structure in bytes. */ -#define BITVEC_SZ 512 + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); -/* Round the union size down to the nearest pointer boundary, since that's how -** it will be aligned within the Bitvec struct. */ -#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) + assert( id!=0 ); + UNUSED_PARAMETER(pVfs); -/* Type of the array "element" for the bitmap representation. -** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. -** Setting this to the "natural word" size of your CPU may improve -** performance. */ -#define BITVEC_TELEM u8 -/* Size, in bits, of the bitmap element. */ -#define BITVEC_SZELEM 8 -/* Number of elements in a bitmap array. */ -#define BITVEC_NELEM (BITVEC_USIZE/sizeof(BITVEC_TELEM)) -/* Number of bits in the bitmap array. */ -#define BITVEC_NBIT (BITVEC_NELEM*BITVEC_SZELEM) + pFile->h = INVALID_HANDLE_VALUE; -/* Number of u32 values in hash table. */ -#define BITVEC_NINT (BITVEC_USIZE/sizeof(u32)) -/* Maximum number of entries in hash table before -** sub-dividing and re-hashing. */ -#define BITVEC_MXHASH (BITVEC_NINT/2) -/* Hashing function for the aHash representation. -** Empirical testing showed that the *37 multiplier -** (an arbitrary prime)in the hash function provided -** no fewer collisions than the no-op *1. */ -#define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zUtf8Name ){ + assert(isDelete && !isOpenJournal); + rc = getTempname(MAX_PATH+1, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zUtf8Name = zTmpname; + } -#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) + /* Convert the filename to the system encoding. */ + zConverted = convertUtf8Filename(zUtf8Name); + if( zConverted==0 ){ + return SQLITE_NOMEM; + } + if( isReadWrite ){ + dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + }else{ + dwDesiredAccess = GENERIC_READ; + } -/* -** A bitmap is an instance of the following structure. -** -** This bitmap records the existance of zero or more bits -** with values between 1 and iSize, inclusive. -** -** There are three possible representations of the bitmap. -** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight -** bitmap. The least significant bit is bit 1. -** -** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is -** a hash table that will hold up to BITVEC_MXHASH distinct values. -** -** Otherwise, the value i is redirected into one of BITVEC_NPTR -** sub-bitmaps pointed to by Bitvec.u.apSub[]. Each subbitmap -** handles up to iDivisor separate values of i. apSub[0] holds -** values between 1 and iDivisor. apSub[1] holds values between -** iDivisor+1 and 2*iDivisor. apSub[N] holds values between -** N*iDivisor+1 and (N+1)*iDivisor. Each subbitmap is normalized -** to hold deal with values between 1 and iDivisor. -*/ -struct Bitvec { - u32 iSize; /* Maximum bit index. Max iSize is 4,294,967,296. */ - u32 nSet; /* Number of bits that are set - only valid for aHash - ** element. Max is BITVEC_NINT. For BITVEC_SZ of 512, - ** this would be 125. */ - u32 iDivisor; /* Number of bits handled by each apSub[] entry. */ - /* Should >=0 for apSub element. */ - /* Max iDivisor is max(u32) / BITVEC_NPTR + 1. */ - /* For a BITVEC_SZ of 512, this would be 34,359,739. */ - union { - BITVEC_TELEM aBitmap[BITVEC_NELEM]; /* Bitmap representation */ - u32 aHash[BITVEC_NINT]; /* Hash table representation */ - Bitvec *apSub[BITVEC_NPTR]; /* Recursive representation */ - } u; -}; + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" + ** as it is usually understood. + */ + if( isExclusive ){ + /* Creates a new file, only if it does not already exist. */ + /* If the file exists, it fails. */ + dwCreationDisposition = CREATE_NEW; + }else if( isCreate ){ + /* Open existing file, or create if it doesn't exist */ + dwCreationDisposition = OPEN_ALWAYS; + }else{ + /* Opens a file, only if it exists. */ + dwCreationDisposition = OPEN_EXISTING; + } -/* -** Create a new bitmap object able to handle bits between 0 and iSize, -** inclusive. Return a pointer to the new object. Return NULL if -** malloc fails. -*/ -SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ - Bitvec *p; - assert( sizeof(*p)==BITVEC_SZ ); - p = sqlite3MallocZero( sizeof(*p) ); - if( p ){ - p->iSize = iSize; + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + + if( isDelete ){ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; + isTemp = 1; +#else + dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY + | FILE_ATTRIBUTE_HIDDEN + | FILE_FLAG_DELETE_ON_CLOSE; +#endif + }else{ + dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; } - return p; -} + /* Reports from the internet are that performance is always + ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; +#endif -/* -** Check to see if the i-th bit is set. Return true or false. -** If p is NULL (if the bitmap has not been created) or if -** i is out of range, then return false. + if( isNT() ){ + h = CreateFileW((WCHAR*)zConverted, + dwDesiredAccess, + dwShareMode, + NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL + ); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. */ -SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ - if( p==0 ) return 0; - if( i>p->iSize || i==0 ) return 0; - i--; - while( p->iDivisor ){ - u32 bin = i/p->iDivisor; - i = i%p->iDivisor; - p = p->u.apSub[bin]; - if (!p) { - return 0; - } +#if SQLITE_OS_WINCE==0 + }else{ + h = CreateFileA((char*)zConverted, + dwDesiredAccess, + dwShareMode, + NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL + ); +#endif } - if( p->iSize<=BITVEC_NBIT ){ - return (p->u.aBitmap[i/BITVEC_SZELEM] & (1<<(i&(BITVEC_SZELEM-1))))!=0; - } else{ - u32 h = BITVEC_HASH(i++); - while( p->u.aHash[h] ){ - if( p->u.aHash[h]==i ) return 1; - h = (h+1) % BITVEC_NINT; + + OSTRACE(("OPEN %d %s 0x%lx %s\n", + h, zName, dwDesiredAccess, + h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + + if( h==INVALID_HANDLE_VALUE ){ + pFile->lastErrno = GetLastError(); + winLogError(SQLITE_CANTOPEN, "winOpen", zUtf8Name); + free(zConverted); + if( isReadWrite ){ + return winOpen(pVfs, zName, id, + ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); + }else{ + return SQLITE_CANTOPEN_BKPT; } - return 0; } -} -/* -** Set the i-th bit. Return 0 on success and an error code if -** anything goes wrong. -** -** This routine might cause sub-bitmaps to be allocated. Failing -** to get the memory needed to hold the sub-bitmap is the only -** that can go wrong with an insert, assuming p and i are valid. -** -** The calling function must ensure that p is a valid Bitvec object -** and that the value for "i" is within range of the Bitvec object. -** Otherwise the behavior is undefined. -*/ -SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ - u32 h; - if( p==0 ) return SQLITE_OK; - assert( i>0 ); - assert( i<=p->iSize ); - i--; - while((p->iSize > BITVEC_NBIT) && p->iDivisor) { - u32 bin = i/p->iDivisor; - i = i%p->iDivisor; - if( p->u.apSub[bin]==0 ){ - p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); - if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + if( pOutFlags ){ + if( isReadWrite ){ + *pOutFlags = SQLITE_OPEN_READWRITE; + }else{ + *pOutFlags = SQLITE_OPEN_READONLY; } - p = p->u.apSub[bin]; - } - if( p->iSize<=BITVEC_NBIT ){ - p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1)); - return SQLITE_OK; } - h = BITVEC_HASH(i++); - /* if there wasn't a hash collision, and this doesn't */ - /* completely fill the hash, then just add it without */ - /* worring about sub-dividing and re-hashing. */ - if( !p->u.aHash[h] ){ - if (p->nSet<(BITVEC_NINT-1)) { - goto bitvec_set_end; - } else { - goto bitvec_set_rehash; - } + + memset(pFile, 0, sizeof(*pFile)); + pFile->pMethod = &winIoMethod; + pFile->h = h; + pFile->lastErrno = NO_ERROR; + pFile->pVfs = pVfs; + pFile->pShm = 0; + pFile->zPath = zName; + pFile->sectorSize = getSectorSize(pVfs, zUtf8Name); + +#if SQLITE_OS_WINCE + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && !winceCreateLock(zName, pFile) + ){ + CloseHandle(h); + free(zConverted); + return SQLITE_CANTOPEN_BKPT; } - /* there was a collision, check to see if it's already */ - /* in hash, if not, try to find a spot for it */ - do { - if( p->u.aHash[h]==i ) return SQLITE_OK; - h++; - if( h>=BITVEC_NINT ) h = 0; - } while( p->u.aHash[h] ); - /* we didn't find it in the hash. h points to the first */ - /* available free spot. check to see if this is going to */ - /* make our hash too "full". */ -bitvec_set_rehash: - if( p->nSet>=BITVEC_MXHASH ){ - unsigned int j; - int rc; - u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); - if( aiValues==0 ){ - return SQLITE_NOMEM; - }else{ - memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); - memset(p->u.apSub, 0, sizeof(p->u.apSub)); - p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; - rc = sqlite3BitvecSet(p, i); - for(j=0; jzDeleteOnClose = zConverted; + }else +#endif + { + free(zConverted); } -bitvec_set_end: - p->nSet++; - p->u.aHash[h] = i; - return SQLITE_OK; + + OpenCounter(+1); + return rc; } /* -** Clear the i-th bit. +** Delete the named file. ** -** pBuf must be a pointer to at least BITVEC_SZ bytes of temporary storage -** that BitvecClear can use to rebuilt its hash table. +** Note that windows does not allow a file to be deleted if some other +** process has it open. Sometimes a virus scanner or indexing program +** will open a journal file shortly after it is created in order to do +** whatever it does. While this other process is holding the +** file open, we will be unable to delete it. To work around this +** problem, we delay 100 milliseconds and try to delete again. Up +** to MX_DELETION_ATTEMPTs deletion attempts are run before giving +** up and returning an error. */ -SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec *p, u32 i, void *pBuf){ - if( p==0 ) return; - assert( i>0 ); - i--; - while( p->iDivisor ){ - u32 bin = i/p->iDivisor; - i = i%p->iDivisor; - p = p->u.apSub[bin]; - if (!p) { - return; - } +#define MX_DELETION_ATTEMPTS 5 +static int winDelete( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on win32 */ +){ + int cnt = 0; + DWORD rc; + DWORD error = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(syncDir); + + SimulateIOError(return SQLITE_IOERR_DELETE); + zConverted = convertUtf8Filename(zFilename); + if( zConverted==0 ){ + return SQLITE_NOMEM; } - if( p->iSize<=BITVEC_NBIT ){ - p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + if( isNT() ){ + do{ + DeleteFileW(zConverted); + }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES) + || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) + && (++cnt < MX_DELETION_ATTEMPTS) + && (Sleep(100), 1) ); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 }else{ - unsigned int j; - u32 *aiValues = pBuf; - memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); - memset(p->u.aHash, 0, sizeof(p->u.aHash)); - p->nSet = 0; - for(j=0; jnSet++; - while( p->u.aHash[h] ){ - h++; - if( h>=BITVEC_NINT ) h = 0; - } - p->u.aHash[h] = aiValues[j]; - } - } + do{ + DeleteFileA(zConverted); + }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES) + || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) + && (++cnt < MX_DELETION_ATTEMPTS) + && (Sleep(100), 1) ); +#endif } + free(zConverted); + OSTRACE(("DELETE \"%s\" %s\n", zFilename, + ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ? + "ok" : "failed" )); + + return ( (rc == INVALID_FILE_ATTRIBUTES) + && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : + winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); } /* -** Destroy a bitmap object. Reclaim all memory used. +** Check the existance and status of a file. */ -SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec *p){ - if( p==0 ) return; - if( p->iDivisor ){ - unsigned int i; - for(i=0; iu.apSub[i]); +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +){ + DWORD attr; + int rc = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + zConverted = convertUtf8Filename(zFilename); + if( zConverted==0 ){ + return SQLITE_NOMEM; + } + if( isNT() ){ + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + if( GetFileAttributesExW((WCHAR*)zConverted, + GetFileExInfoStandard, + &sAttrData) ){ + /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file + ** as if it does not exist. + */ + if( flags==SQLITE_ACCESS_EXISTS + && sAttrData.nFileSizeHigh==0 + && sAttrData.nFileSizeLow==0 ){ + attr = INVALID_FILE_ATTRIBUTES; + }else{ + attr = sAttrData.dwFileAttributes; + } + }else{ + if( GetLastError()!=ERROR_FILE_NOT_FOUND ){ + winLogError(SQLITE_IOERR_ACCESS, "winAccess", zFilename); + free(zConverted); + return SQLITE_IOERR_ACCESS; + }else{ + attr = INVALID_FILE_ATTRIBUTES; + } } +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + attr = GetFileAttributesA((char*)zConverted); +#endif } - sqlite3_free(p); + free(zConverted); + switch( flags ){ + case SQLITE_ACCESS_READ: + case SQLITE_ACCESS_EXISTS: + rc = attr!=INVALID_FILE_ATTRIBUTES; + break; + case SQLITE_ACCESS_READWRITE: + rc = (attr & FILE_ATTRIBUTE_READONLY)==0; + break; + default: + assert(!"Invalid flags argument"); + } + *pResOut = rc; + return SQLITE_OK; } -/* -** Return the value of the iSize parameter specified when Bitvec *p -** was created. -*/ -SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ - return p->iSize; -} -#ifndef SQLITE_OMIT_BUILTIN_TEST /* -** Let V[] be an array of unsigned characters sufficient to hold -** up to N bits. Let I be an integer between 0 and N. 0<=ImxPathname +** bytes in size. */ -#define SETBIT(V,I) V[I>>3] |= (1<<(I&7)) -#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) -#define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + +#if defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(nFull); + cygwin_conv_to_full_win32_path(zRelative, zFull); + return SQLITE_OK; +#endif -/* -** This routine runs an extensive test of the Bitvec code. -** -** The input is an array of integers that acts as a program -** to test the Bitvec. The integers are opcodes followed -** by 0, 1, or 3 operands, depending on the opcode. Another -** opcode follows immediately after the last operand. -** -** There are 6 opcodes numbered from 0 through 5. 0 is the -** "halt" opcode and causes the test to end. -** -** 0 Halt and return the number of errors -** 1 N S X Set N bits beginning with S and incrementing by X -** 2 N S X Clear N bits beginning with S and incrementing by X -** 3 N Set N randomly chosen bits -** 4 N Clear N randomly chosen bits -** 5 N S X Set N bits from S increment X in array only, not in bitvec -** -** The opcodes 1 through 4 perform set and clear operations are performed -** on both a Bitvec object and on a linear array of bits obtained from malloc. -** Opcode 5 works on the linear array only, not on the Bitvec. -** Opcode 5 is used to deliberately induce a fault in order to -** confirm that error detection works. -** -** At the conclusion of the test the linear array is compared -** against the Bitvec object. If there are any differences, -** an error is returned. If they are the same, zero is returned. -** -** If a memory allocation error occurs, return -1. -*/ -SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ - Bitvec *pBitvec = 0; - unsigned char *pV = 0; - int rc = -1; - int i, nx, pc, op; - void *pTmpSpace; +#if SQLITE_OS_WINCE + SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(nFull); + /* WinCE has no concept of a relative pathname, or so I am told. */ + sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zRelative); + return SQLITE_OK; +#endif - /* Allocate the Bitvec to be tested and a linear array of - ** bits to act as the reference */ - pBitvec = sqlite3BitvecCreate( sz ); - pV = sqlite3_malloc( (sz+7)/8 + 1 ); - pTmpSpace = sqlite3_malloc(BITVEC_SZ); - if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; - memset(pV, 0, (sz+7)/8 + 1); +#if !SQLITE_OS_WINCE && !defined(__CYGWIN__) + int nByte; + void *zConverted; + char *zOut; - /* NULL pBitvec tests */ - sqlite3BitvecSet(0, 1); - sqlite3BitvecClear(0, 1, pTmpSpace); + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && sqlite3Isalpha(zRelative[1]) && zRelative[2]==':' ){ + zRelative++; + } - /* Run the program */ - pc = 0; - while( (op = aOp[pc])!=0 ){ - switch( op ){ - case 1: - case 2: - case 5: { - nx = 4; - i = aOp[pc+2] - 1; - aOp[pc+2] += aOp[pc+3]; - break; - } - case 3: - case 4: - default: { - nx = 2; - sqlite3_randomness(sizeof(i), &i); - break; - } - } - if( (--aOp[pc+1]) > 0 ) nx = 0; - pc += nx; - i = (i & 0x7fffffff)%sz; - if( (op & 1)!=0 ){ - SETBIT(pV, (i+1)); - if( op!=5 ){ - if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; - } - }else{ - CLEARBIT(pV, (i+1)); - sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(nFull); + zConverted = convertUtf8Filename(zRelative); + if( isNT() ){ + WCHAR *zTemp; + nByte = GetFullPathNameW((WCHAR*)zConverted, 0, 0, 0) + 3; + zTemp = malloc( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + free(zConverted); + return SQLITE_NOMEM; + } + GetFullPathNameW((WCHAR*)zConverted, nByte, zTemp, 0); + free(zConverted); + zOut = unicodeToUtf8(zTemp); + free(zTemp); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + char *zTemp; + nByte = GetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; + zTemp = malloc( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + free(zConverted); + return SQLITE_NOMEM; } + GetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + free(zConverted); + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + free(zTemp); +#endif + } + if( zOut ){ + sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zOut); + free(zOut); + return SQLITE_OK; + }else{ + return SQLITE_NOMEM; } +#endif +} - /* Test to make sure the linear array exactly matches the - ** Bitvec object. Start with the assumption that they do - ** match (rc==0). Change rc to non-zero if a discrepancy - ** is found. +/* +** Get the sector size of the device used to store +** file. +*/ +static int getSectorSize( + sqlite3_vfs *pVfs, + const char *zRelative /* UTF-8 file name */ +){ + DWORD bytesPerSector = SQLITE_DEFAULT_SECTOR_SIZE; + /* GetDiskFreeSpace is not supported under WINCE */ +#if SQLITE_OS_WINCE + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(zRelative); +#else + char zFullpath[MAX_PATH+1]; + int rc; + DWORD dwRet = 0; + DWORD dwDummy; + + /* + ** We need to get the full path name of the file + ** to get the drive letter to look up the sector + ** size. */ - rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) - + sqlite3BitvecTest(pBitvec, 0) - + (sqlite3BitvecSize(pBitvec) - sz); - for(i=1; i<=sz; i++){ - if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ - rc = i; - break; + SimulateIOErrorBenign(1); + rc = winFullPathname(pVfs, zRelative, MAX_PATH, zFullpath); + SimulateIOErrorBenign(0); + if( rc == SQLITE_OK ) + { + void *zConverted = convertUtf8Filename(zFullpath); + if( zConverted ){ + if( isNT() ){ + /* trim path to just drive reference */ + WCHAR *p = zConverted; + for(;*p;p++){ + if( *p == '\\' ){ + *p = '\0'; + break; + } + } + dwRet = GetDiskFreeSpaceW((WCHAR*)zConverted, + &dwDummy, + &bytesPerSector, + &dwDummy, + &dwDummy); + }else{ + /* trim path to just drive reference */ + char *p = (char *)zConverted; + for(;*p;p++){ + if( *p == '\\' ){ + *p = '\0'; + break; + } + } + dwRet = GetDiskFreeSpaceA((char*)zConverted, + &dwDummy, + &bytesPerSector, + &dwDummy, + &dwDummy); + } + free(zConverted); + } + if( !dwRet ){ + bytesPerSector = SQLITE_DEFAULT_SECTOR_SIZE; } } - - /* Free allocated structure */ -bitvec_end: - sqlite3_free(pTmpSpace); - sqlite3_free(pV); - sqlite3BitvecDestroy(pBitvec); - return rc; +#endif + return (int) bytesPerSector; } -#endif /* SQLITE_OMIT_BUILTIN_TEST */ -/************** End of bitvec.c **********************************************/ -/************** Begin file pcache.c ******************************************/ +#ifndef SQLITE_OMIT_LOAD_EXTENSION /* -** 2008 August 05 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file implements that page cache. +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. */ - /* -** A complete page cache is an instance of this structure. +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. */ -struct PCache { - PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ - PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ - int nMax; /* Configured cache size */ - int szPage; /* Size of every page in this cache */ - int szExtra; /* Size of extra space for each page */ - int bPurgeable; /* True if pages are on backing store */ - int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ - void *pStress; /* Argument to xStress */ - sqlite3_pcache *pCache; /* Pluggable cache module */ - PgHdr *pPage1; /* Reference to page 1 */ -}; - -/* -** Some of the assert() macros in this code are too expensive to run -** even during normal debugging. Use them only rarely on long-running -** tests. Enable the expensive asserts using the -** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option. +static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ + HANDLE h; + void *zConverted = convertUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); + if( zConverted==0 ){ + return 0; + } + if( isNT() ){ + h = LoadLibraryW((WCHAR*)zConverted); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. */ -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT -# define expensive_assert(X) assert(X) +#if SQLITE_OS_WINCE==0 + }else{ + h = LoadLibraryA((char*)zConverted); +#endif + } + free(zConverted); + return (void*)h; +} +static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ + UNUSED_PARAMETER(pVfs); + getLastErrorMsg(nBuf, zBufOut); +} +void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ + UNUSED_PARAMETER(pVfs); +#if SQLITE_OS_WINCE + /* The GetProcAddressA() routine is only available on wince. */ + return (void(*)(void))GetProcAddressA((HANDLE)pHandle, zSymbol); #else -# define expensive_assert(X) + /* All other windows platforms expect GetProcAddress() to take + ** an Ansi string regardless of the _UNICODE setting */ + return (void(*)(void))GetProcAddress((HANDLE)pHandle, zSymbol); +#endif +} +void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ + UNUSED_PARAMETER(pVfs); + FreeLibrary((HANDLE)pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define winDlOpen 0 + #define winDlError 0 + #define winDlSym 0 + #define winDlClose 0 #endif -/********************************** Linked List Management ********************/ -#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) /* -** Check that the pCache->pSynced variable is set correctly. If it -** is not, either fail an assert or return zero. Otherwise, return -** non-zero. This is only used in debugging builds, as follows: -** -** expensive_assert( pcacheCheckSynced(pCache) ); +** Write up to nBuf bytes of randomness into zBuf. */ -static int pcacheCheckSynced(PCache *pCache){ - PgHdr *p; - for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ - assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); +static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + int n = 0; + UNUSED_PARAMETER(pVfs); +#if defined(SQLITE_TEST) + n = nBuf; + memset(zBuf, 0, nBuf); +#else + if( sizeof(SYSTEMTIME)<=nBuf-n ){ + SYSTEMTIME x; + GetSystemTime(&x); + memcpy(&zBuf[n], &x, sizeof(x)); + n += sizeof(x); } - return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); -} -#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ - -/* -** Remove page pPage from the list of dirty pages. -*/ -static void pcacheRemoveFromDirtyList(PgHdr *pPage){ - PCache *p = pPage->pCache; - - assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); - assert( pPage->pDirtyPrev || pPage==p->pDirty ); - - /* Update the PCache1.pSynced variable if necessary. */ - if( p->pSynced==pPage ){ - PgHdr *pSynced = pPage->pDirtyPrev; - while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ - pSynced = pSynced->pDirtyPrev; - } - p->pSynced = pSynced; + if( sizeof(DWORD)<=nBuf-n ){ + DWORD pid = GetCurrentProcessId(); + memcpy(&zBuf[n], &pid, sizeof(pid)); + n += sizeof(pid); } - - if( pPage->pDirtyNext ){ - pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; - }else{ - assert( pPage==p->pDirtyTail ); - p->pDirtyTail = pPage->pDirtyPrev; + if( sizeof(DWORD)<=nBuf-n ){ + DWORD cnt = GetTickCount(); + memcpy(&zBuf[n], &cnt, sizeof(cnt)); + n += sizeof(cnt); } - if( pPage->pDirtyPrev ){ - pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; - }else{ - assert( pPage==p->pDirty ); - p->pDirty = pPage->pDirtyNext; + if( sizeof(LARGE_INTEGER)<=nBuf-n ){ + LARGE_INTEGER i; + QueryPerformanceCounter(&i); + memcpy(&zBuf[n], &i, sizeof(i)); + n += sizeof(i); } - pPage->pDirtyNext = 0; - pPage->pDirtyPrev = 0; - - expensive_assert( pcacheCheckSynced(p) ); +#endif + return n; } + /* -** Add page pPage to the head of the dirty list (PCache1.pDirty is set to -** pPage). +** Sleep for a little while. Return the amount of time slept. */ -static void pcacheAddToDirtyList(PgHdr *pPage){ - PCache *p = pPage->pCache; - - assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); - - pPage->pDirtyNext = p->pDirty; - if( pPage->pDirtyNext ){ - assert( pPage->pDirtyNext->pDirtyPrev==0 ); - pPage->pDirtyNext->pDirtyPrev = pPage; - } - p->pDirty = pPage; - if( !p->pDirtyTail ){ - p->pDirtyTail = pPage; - } - if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ - p->pSynced = pPage; - } - expensive_assert( pcacheCheckSynced(p) ); +static int winSleep(sqlite3_vfs *pVfs, int microsec){ + Sleep((microsec+999)/1000); + UNUSED_PARAMETER(pVfs); + return ((microsec+999)/1000)*1000; } /* -** Wrapper around the pluggable caches xUnpin method. If the cache is -** being used for an in-memory database, this function is a no-op. +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. */ -static void pcacheUnpin(PgHdr *p){ - PCache *pCache = p->pCache; - if( pCache->bPurgeable ){ - if( p->pgno==1 ){ - pCache->pPage1 = 0; - } - sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 0); - } -} +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif -/*************************************************** General Interfaces ****** +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. ** -** Initialize and shutdown the page cache subsystem. Neither of these -** functions are threadsafe. +** On success, return 0. Return 1 if the time and date cannot be found. */ -SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ - if( sqlite3GlobalConfig.pcache.xInit==0 ){ - /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the - ** built-in default page cache is used instead of the application defined - ** page cache. */ - sqlite3PCacheSetDefault(); +static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ + /* FILETIME structure is a 64-bit value representing the number of + 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). + */ + FILETIME ft; + static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; +#ifdef SQLITE_TEST + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; +#endif + /* 2^32 - to avoid use of LL and warnings in gcc */ + static const sqlite3_int64 max32BitValue = + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; + +#if SQLITE_OS_WINCE + SYSTEMTIME time; + GetSystemTime(&time); + /* if SystemTimeToFileTime() fails, it returns zero. */ + if (!SystemTimeToFileTime(&time,&ft)){ + return 1; } - return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); -} -SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ - if( sqlite3GlobalConfig.pcache.xShutdown ){ - /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ - sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); +#else + GetSystemTimeAsFileTime( &ft ); +#endif + + *piNow = winFiletimeEpoch + + ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + + (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; } +#endif + UNUSED_PARAMETER(pVfs); + return 0; } /* -** Return the size in bytes of a PCache object. -*/ -SQLITE_PRIVATE int sqlite3PcacheSize(void){ return sizeof(PCache); } - -/* -** Create a new PCache object. Storage space to hold the object -** has already been allocated and is passed in as the p pointer. -** The caller discovers how much space needs to be allocated by -** calling sqlite3PcacheSize(). +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. */ -SQLITE_PRIVATE void sqlite3PcacheOpen( - int szPage, /* Size of every page */ - int szExtra, /* Extra space associated with each page */ - int bPurgeable, /* True if pages are on backing store */ - int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */ - void *pStress, /* Argument to xStress */ - PCache *p /* Preallocated space for the PCache */ -){ - memset(p, 0, sizeof(PCache)); - p->szPage = szPage; - p->szExtra = szExtra; - p->bPurgeable = bPurgeable; - p->xStress = xStress; - p->pStress = pStress; - p->nMax = 100; +int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ + int rc; + sqlite3_int64 i; + rc = winCurrentTimeInt64(pVfs, &i); + if( !rc ){ + *prNow = i/86400000.0; + } + return rc; } /* -** Change the page size for PCache object. The caller must ensure that there -** are no outstanding page references when this function is called. +** The idea is that this function works like a combination of +** GetLastError() and FormatMessage() on windows (or errno and +** strerror_r() on unix). After an error is returned by an OS +** function, SQLite calls this function with zBuf pointing to +** a buffer of nBuf bytes. The OS layer should populate the +** buffer with a nul-terminated UTF-8 encoded error message +** describing the last IO error to have occurred within the calling +** thread. +** +** If the error message is too large for the supplied buffer, +** it should be truncated. The return value of xGetLastError +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). If non-zero is returned, +** then it is not necessary to include the nul-terminator character +** in the output buffer. +** +** Not supplying an error message will have no adverse effect +** on SQLite. It is fine to have an implementation that never +** returns an error message: +** +** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +** assert(zBuf[0]=='\0'); +** return 0; +** } +** +** However if an error message is supplied, it will be incorporated +** by sqlite into the error message available to the user using +** sqlite3_errmsg(), possibly making IO errors easier to debug. */ -SQLITE_PRIVATE void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); - pCache->pCache = 0; - pCache->pPage1 = 0; - } - pCache->szPage = szPage; +static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + UNUSED_PARAMETER(pVfs); + return getLastErrorMsg(nBuf, zBuf); } + + /* -** Try to obtain a page from the cache. +** Initialize and deinitialize the operating system interface. */ -SQLITE_PRIVATE int sqlite3PcacheFetch( - PCache *pCache, /* Obtain the page from this cache */ - Pgno pgno, /* Page number to obtain */ - int createFlag, /* If true, create page if it does not exist already */ - PgHdr **ppPage /* Write the page here */ -){ - PgHdr *pPage = 0; - int eCreate; +SQLITE_API int sqlite3_os_init(void){ + static sqlite3_vfs winVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + MAX_PATH, /* mxPathname */ + 0, /* pNext */ + "win32", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + 0, /* xSetSystemCall */ + 0, /* xGetSystemCall */ + 0, /* xNextSystemCall */ + }; - assert( pCache!=0 ); - assert( createFlag==1 || createFlag==0 ); - assert( pgno>0 ); +#ifndef SQLITE_OMIT_WAL + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&winSysInfo); + assert(winSysInfo.dwAllocationGranularity > 0); +#endif - /* If the pluggable cache (sqlite3_pcache*) has not been allocated, - ** allocate it now. - */ - if( !pCache->pCache && createFlag ){ - sqlite3_pcache *p; - int nByte; - nByte = pCache->szPage + pCache->szExtra + sizeof(PgHdr); - p = sqlite3GlobalConfig.pcache.xCreate(nByte, pCache->bPurgeable); - if( !p ){ - return SQLITE_NOMEM; - } - sqlite3GlobalConfig.pcache.xCachesize(p, pCache->nMax); - pCache->pCache = p; - } + sqlite3_vfs_register(&winVfs, 1); + return SQLITE_OK; +} +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; +} - eCreate = createFlag * (1 + (!pCache->bPurgeable || !pCache->pDirty)); - if( pCache->pCache ){ - pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, eCreate); - } +#endif /* SQLITE_OS_WIN */ - if( !pPage && eCreate==1 ){ - PgHdr *pPg; +/************** End of os_win.c **********************************************/ +/************** Begin file bitvec.c ******************************************/ +/* +** 2008 February 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an object that represents a fixed-length +** bitmap. Bits are numbered starting with 1. +** +** A bitmap is used to record which pages of a database file have been +** journalled during a transaction, or which pages have the "dont-write" +** property. Usually only a few pages are meet either condition. +** So the bitmap is usually sparse and has low cardinality. +** But sometimes (for example when during a DROP of a large table) most +** or all of the pages in a database can get journalled. In those cases, +** the bitmap becomes dense with high cardinality. The algorithm needs +** to handle both cases well. +** +** The size of the bitmap is fixed when the object is created. +** +** All bits are clear when the bitmap is created. Individual bits +** may be set or cleared one at a time. +** +** Test operations are about 100 times more common that set operations. +** Clear operations are exceedingly rare. There are usually between +** 5 and 500 set operations per Bitvec object, though the number of sets can +** sometimes grow into tens of thousands or larger. The size of the +** Bitvec object is the number of pages in the database file at the +** start of a transaction, and is thus usually less than a few thousand, +** but can be as large as 2 billion for a really big database. +*/ - /* Find a dirty page to write-out and recycle. First try to find a - ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC - ** cleared), but if that is not possible settle for any other - ** unreferenced dirty page. - */ - expensive_assert( pcacheCheckSynced(pCache) ); - for(pPg=pCache->pSynced; - pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); - pPg=pPg->pDirtyPrev - ); - pCache->pSynced = pPg; - if( !pPg ){ - for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); - } - if( pPg ){ - int rc; - rc = pCache->xStress(pCache->pStress, pPg); - if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - return rc; - } - } +/* Size of the Bitvec structure in bytes. */ +#define BITVEC_SZ 512 - pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, 2); - } +/* Round the union size down to the nearest pointer boundary, since that's how +** it will be aligned within the Bitvec struct. */ +#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) - if( pPage ){ - if( !pPage->pData ){ - memset(pPage, 0, sizeof(PgHdr)); - pPage->pData = (void *)&pPage[1]; - pPage->pExtra = (void*)&((char *)pPage->pData)[pCache->szPage]; - memset(pPage->pExtra, 0, pCache->szExtra); - pPage->pCache = pCache; - pPage->pgno = pgno; - } - assert( pPage->pCache==pCache ); - assert( pPage->pgno==pgno ); - assert( pPage->pData==(void *)&pPage[1] ); - assert( pPage->pExtra==(void *)&((char *)&pPage[1])[pCache->szPage] ); +/* Type of the array "element" for the bitmap representation. +** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. +** Setting this to the "natural word" size of your CPU may improve +** performance. */ +#define BITVEC_TELEM u8 +/* Size, in bits, of the bitmap element. */ +#define BITVEC_SZELEM 8 +/* Number of elements in a bitmap array. */ +#define BITVEC_NELEM (BITVEC_USIZE/sizeof(BITVEC_TELEM)) +/* Number of bits in the bitmap array. */ +#define BITVEC_NBIT (BITVEC_NELEM*BITVEC_SZELEM) - if( 0==pPage->nRef ){ - pCache->nRef++; - } - pPage->nRef++; - if( pgno==1 ){ - pCache->pPage1 = pPage; - } - } - *ppPage = pPage; - return (pPage==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; -} +/* Number of u32 values in hash table. */ +#define BITVEC_NINT (BITVEC_USIZE/sizeof(u32)) +/* Maximum number of entries in hash table before +** sub-dividing and re-hashing. */ +#define BITVEC_MXHASH (BITVEC_NINT/2) +/* Hashing function for the aHash representation. +** Empirical testing showed that the *37 multiplier +** (an arbitrary prime)in the hash function provided +** no fewer collisions than the no-op *1. */ +#define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) -/* -** Decrement the reference count on a page. If the page is clean and the -** reference count drops to 0, then it is made elible for recycling. -*/ -SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr *p){ - assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - PCache *pCache = p->pCache; - pCache->nRef--; - if( (p->flags&PGHDR_DIRTY)==0 ){ - pcacheUnpin(p); - }else{ - /* Move the page to the head of the dirty list. */ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); - } - } -} +#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) -/* -** Increase the reference count of a supplied page by 1. -*/ -SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ - assert(p->nRef>0); - p->nRef++; -} /* -** Drop a page from the cache. There must be exactly one reference to the -** page. This function deletes that reference, so after it returns the -** page pointed to by p is invalid. +** A bitmap is an instance of the following structure. +** +** This bitmap records the existance of zero or more bits +** with values between 1 and iSize, inclusive. +** +** There are three possible representations of the bitmap. +** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight +** bitmap. The least significant bit is bit 1. +** +** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is +** a hash table that will hold up to BITVEC_MXHASH distinct values. +** +** Otherwise, the value i is redirected into one of BITVEC_NPTR +** sub-bitmaps pointed to by Bitvec.u.apSub[]. Each subbitmap +** handles up to iDivisor separate values of i. apSub[0] holds +** values between 1 and iDivisor. apSub[1] holds values between +** iDivisor+1 and 2*iDivisor. apSub[N] holds values between +** N*iDivisor+1 and (N+1)*iDivisor. Each subbitmap is normalized +** to hold deal with values between 1 and iDivisor. */ -SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ - PCache *pCache; - assert( p->nRef==1 ); - if( p->flags&PGHDR_DIRTY ){ - pcacheRemoveFromDirtyList(p); - } - pCache = p->pCache; - pCache->nRef--; - if( p->pgno==1 ){ - pCache->pPage1 = 0; - } - sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 1); -} +struct Bitvec { + u32 iSize; /* Maximum bit index. Max iSize is 4,294,967,296. */ + u32 nSet; /* Number of bits that are set - only valid for aHash + ** element. Max is BITVEC_NINT. For BITVEC_SZ of 512, + ** this would be 125. */ + u32 iDivisor; /* Number of bits handled by each apSub[] entry. */ + /* Should >=0 for apSub element. */ + /* Max iDivisor is max(u32) / BITVEC_NPTR + 1. */ + /* For a BITVEC_SZ of 512, this would be 34,359,739. */ + union { + BITVEC_TELEM aBitmap[BITVEC_NELEM]; /* Bitmap representation */ + u32 aHash[BITVEC_NINT]; /* Hash table representation */ + Bitvec *apSub[BITVEC_NPTR]; /* Recursive representation */ + } u; +}; /* -** Make sure the page is marked as dirty. If it isn't dirty already, -** make it so. +** Create a new bitmap object able to handle bits between 0 and iSize, +** inclusive. Return a pointer to the new object. Return NULL if +** malloc fails. */ -SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ - p->flags &= ~PGHDR_DONT_WRITE; - assert( p->nRef>0 ); - if( 0==(p->flags & PGHDR_DIRTY) ){ - p->flags |= PGHDR_DIRTY; - pcacheAddToDirtyList( p); +SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ + Bitvec *p; + assert( sizeof(*p)==BITVEC_SZ ); + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->iSize = iSize; } + return p; } /* -** Make sure the page is marked as clean. If it isn't clean already, -** make it so. +** Check to see if the i-th bit is set. Return true or false. +** If p is NULL (if the bitmap has not been created) or if +** i is out of range, then return false. */ -SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ - if( (p->flags & PGHDR_DIRTY) ){ - pcacheRemoveFromDirtyList(p); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); - if( p->nRef==0 ){ - pcacheUnpin(p); +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + if( p==0 ) return 0; + if( i>p->iSize || i==0 ) return 0; + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return 0; } } -} - -/* -** Make every page in the cache clean. -*/ -SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache *pCache){ - PgHdr *p; - while( (p = pCache->pDirty)!=0 ){ - sqlite3PcacheMakeClean(p); + if( p->iSize<=BITVEC_NBIT ){ + return (p->u.aBitmap[i/BITVEC_SZELEM] & (1<<(i&(BITVEC_SZELEM-1))))!=0; + } else{ + u32 h = BITVEC_HASH(i++); + while( p->u.aHash[h] ){ + if( p->u.aHash[h]==i ) return 1; + h = (h+1) % BITVEC_NINT; + } + return 0; } } /* -** Clear the PGHDR_NEED_SYNC flag from all dirty pages. +** Set the i-th bit. Return 0 on success and an error code if +** anything goes wrong. +** +** This routine might cause sub-bitmaps to be allocated. Failing +** to get the memory needed to hold the sub-bitmap is the only +** that can go wrong with an insert, assuming p and i are valid. +** +** The calling function must ensure that p is a valid Bitvec object +** and that the value for "i" is within range of the Bitvec object. +** Otherwise the behavior is undefined. */ -SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ - PgHdr *p; - for(p=pCache->pDirty; p; p=p->pDirtyNext){ - p->flags &= ~PGHDR_NEED_SYNC; +SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ + u32 h; + if( p==0 ) return SQLITE_OK; + assert( i>0 ); + assert( i<=p->iSize ); + i--; + while((p->iSize > BITVEC_NBIT) && p->iDivisor) { + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + if( p->u.apSub[bin]==0 ){ + p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + } + p = p->u.apSub[bin]; } - pCache->pSynced = pCache->pDirtyTail; -} - -/* -** Change the page number of page p to newPgno. -*/ -SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ - PCache *pCache = p->pCache; - assert( p->nRef>0 ); - assert( newPgno>0 ); - sqlite3GlobalConfig.pcache.xRekey(pCache->pCache, p, p->pgno, newPgno); - p->pgno = newPgno; - if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1)); + return SQLITE_OK; + } + h = BITVEC_HASH(i++); + /* if there wasn't a hash collision, and this doesn't */ + /* completely fill the hash, then just add it without */ + /* worring about sub-dividing and re-hashing. */ + if( !p->u.aHash[h] ){ + if (p->nSet<(BITVEC_NINT-1)) { + goto bitvec_set_end; + } else { + goto bitvec_set_rehash; + } + } + /* there was a collision, check to see if it's already */ + /* in hash, if not, try to find a spot for it */ + do { + if( p->u.aHash[h]==i ) return SQLITE_OK; + h++; + if( h>=BITVEC_NINT ) h = 0; + } while( p->u.aHash[h] ); + /* we didn't find it in the hash. h points to the first */ + /* available free spot. check to see if this is going to */ + /* make our hash too "full". */ +bitvec_set_rehash: + if( p->nSet>=BITVEC_MXHASH ){ + unsigned int j; + int rc; + u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); + if( aiValues==0 ){ + return SQLITE_NOMEM; + }else{ + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.apSub, 0, sizeof(p->u.apSub)); + p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + rc = sqlite3BitvecSet(p, i); + for(j=0; jnSet++; + p->u.aHash[h] = i; + return SQLITE_OK; } /* -** Drop every cache entry whose page number is greater than "pgno". The -** caller must ensure that there are no outstanding references to any pages -** other than page 1 with a page number greater than pgno. +** Clear the i-th bit. ** -** If there is a reference to page 1 and the pgno parameter passed to this -** function is 0, then the data area associated with page 1 is zeroed, but -** the page object is not dropped. +** pBuf must be a pointer to at least BITVEC_SZ bytes of temporary storage +** that BitvecClear can use to rebuilt its hash table. */ -SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ - if( pCache->pCache ){ - PgHdr *p; - PgHdr *pNext; - for(p=pCache->pDirty; p; p=pNext){ - pNext = p->pDirtyNext; - /* This routine never gets call with a positive pgno except right - ** after sqlite3PcacheCleanAll(). So if there are dirty pages, - ** it must be that pgno==0. - */ - assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ - assert( p->flags&PGHDR_DIRTY ); - sqlite3PcacheMakeClean(p); - } +SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec *p, u32 i, void *pBuf){ + if( p==0 ) return; + assert( i>0 ); + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return; } - if( pgno==0 && pCache->pPage1 ){ - memset(pCache->pPage1->pData, 0, pCache->szPage); - pgno = 1; + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + }else{ + unsigned int j; + u32 *aiValues = pBuf; + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.aHash, 0, sizeof(p->u.aHash)); + p->nSet = 0; + for(j=0; jnSet++; + while( p->u.aHash[h] ){ + h++; + if( h>=BITVEC_NINT ) h = 0; + } + p->u.aHash[h] = aiValues[j]; + } } - sqlite3GlobalConfig.pcache.xTruncate(pCache->pCache, pgno+1); } } /* -** Close a cache. +** Destroy a bitmap object. Reclaim all memory used. */ -SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); +SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec *p){ + if( p==0 ) return; + if( p->iDivisor ){ + unsigned int i; + for(i=0; iu.apSub[i]); + } } + sqlite3_free(p); } -/* -** Discard the contents of the cache. +/* +** Return the value of the iSize parameter specified when Bitvec *p +** was created. */ -SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ - sqlite3PcacheTruncate(pCache, 0); +SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ + return p->iSize; } +#ifndef SQLITE_OMIT_BUILTIN_TEST /* -** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pDirtyPrev pointers. +** Let V[] be an array of unsigned characters sufficient to hold +** up to N bits. Let I be an integer between 0 and N. 0<=Ipgnopgno ){ - pTail->pDirty = pA; - pTail = pA; - pA = pA->pDirty; - }else{ - pTail->pDirty = pB; - pTail = pB; - pB = pB->pDirty; - } - } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } - return result.pDirty; -} +#define SETBIT(V,I) V[I>>3] |= (1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 /* -** Sort the list of pages in accending order by pgno. Pages are -** connected by pDirty pointers. The pDirtyPrev pointers are -** corrupted by this sort. +** This routine runs an extensive test of the Bitvec code. ** -** Since there cannot be more than 2^31 distinct pages in a database, -** there cannot be more than 31 buckets required by the merge sorter. -** One extra bucket is added to catch overflow in case something -** ever changes to make the previous sentence incorrect. +** The input is an array of integers that acts as a program +** to test the Bitvec. The integers are opcodes followed +** by 0, 1, or 3 operands, depending on the opcode. Another +** opcode follows immediately after the last operand. +** +** There are 6 opcodes numbered from 0 through 5. 0 is the +** "halt" opcode and causes the test to end. +** +** 0 Halt and return the number of errors +** 1 N S X Set N bits beginning with S and incrementing by X +** 2 N S X Clear N bits beginning with S and incrementing by X +** 3 N Set N randomly chosen bits +** 4 N Clear N randomly chosen bits +** 5 N S X Set N bits from S increment X in array only, not in bitvec +** +** The opcodes 1 through 4 perform set and clear operations are performed +** on both a Bitvec object and on a linear array of bits obtained from malloc. +** Opcode 5 works on the linear array only, not on the Bitvec. +** Opcode 5 is used to deliberately induce a fault in order to +** confirm that error detection works. +** +** At the conclusion of the test the linear array is compared +** against the Bitvec object. If there are any differences, +** an error is returned. If they are the same, zero is returned. +** +** If a memory allocation error occurs, return -1. */ -#define N_SORT_BUCKET 32 -static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ - PgHdr *a[N_SORT_BUCKET], *p; - int i; - memset(a, 0, sizeof(a)); - while( pIn ){ - p = pIn; - pIn = p->pDirty; - p->pDirty = 0; - for(i=0; ALWAYS(i 0 ) nx = 0; + pc += nx; + i = (i & 0x7fffffff)%sz; + if( (op & 1)!=0 ){ + SETBIT(pV, (i+1)); + if( op!=5 ){ + if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; + } + }else{ + CLEARBIT(pV, (i+1)); + sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); } } - p = a[0]; - for(i=1; ipDirty; p; p=p->pDirtyNext){ - p->pDirty = p->pDirtyNext; - } - return pcacheSortDirtyList(pCache->pDirty); -} - -/* -** Return the total number of referenced pages held by the cache. -*/ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ - return pCache->nRef; -} - -/* -** Return the number of references to the page supplied as an argument. -*/ -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ - return p->nRef; -} - -/* -** Return the total number of pages in the cache. -*/ -SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ - int nPage = 0; - if( pCache->pCache ){ - nPage = sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache); - } - return nPage; -} - -#ifdef SQLITE_TEST -/* -** Get the suggested cache-size value. -*/ -SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *pCache){ - return pCache->nMax; -} -#endif -/* -** Set the suggested cache-size value. -*/ -SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ - pCache->nMax = mxPage; - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache.xCachesize(pCache->pCache, mxPage); + /* Test to make sure the linear array exactly matches the + ** Bitvec object. Start with the assumption that they do + ** match (rc==0). Change rc to non-zero if a discrepancy + ** is found. + */ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } } -} -#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) -/* -** For all dirty pages currently in the cache, invoke the specified -** callback. This is only used if the SQLITE_CHECK_PAGES macro is -** defined. -*/ -SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)){ - PgHdr *pDirty; - for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext){ - xIter(pDirty); - } + /* Free allocated structure */ +bitvec_end: + sqlite3_free(pTmpSpace); + sqlite3_free(pV); + sqlite3BitvecDestroy(pBitvec); + return rc; } -#endif +#endif /* SQLITE_OMIT_BUILTIN_TEST */ -/************** End of pcache.c **********************************************/ -/************** Begin file pcache1.c *****************************************/ +/************** End of bitvec.c **********************************************/ +/************** Begin file pcache.c ******************************************/ /* -** 2008 November 05 +** 2008 August 05 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -33323,850 +34693,593 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** May you share freely, never taking more than you give. ** ************************************************************************* -** -** This file implements the default page cache implementation (the -** sqlite3_pcache interface). It also contains part of the implementation -** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. -** If the default page cache implementation is overriden, then neither of -** these two features are available. -*/ - - -typedef struct PCache1 PCache1; -typedef struct PgHdr1 PgHdr1; -typedef struct PgFreeslot PgFreeslot; - -/* Each page cache is an instance of the following object. Every -** open database file (including each in-memory database and each -** temporary or transient database) has a single page cache which -** is an instance of this object. -** -** Pointers to structures of this type are cast and returned as -** opaque sqlite3_pcache* handles. -*/ -struct PCache1 { - /* Cache configuration parameters. Page size (szPage) and the purgeable - ** flag (bPurgeable) are set when the cache is created. nMax may be - ** modified at any time by a call to the pcache1CacheSize() method. - ** The global mutex must be held when accessing nMax. - */ - int szPage; /* Size of allocated pages in bytes */ - int bPurgeable; /* True if cache is purgeable */ - unsigned int nMin; /* Minimum number of pages reserved */ - unsigned int nMax; /* Configured "cache_size" value */ - - /* Hash table of all pages. The following variables may only be accessed - ** when the accessor is holding the global mutex (see pcache1EnterMutex() - ** and pcache1LeaveMutex()). - */ - unsigned int nRecyclable; /* Number of pages in the LRU list */ - unsigned int nPage; /* Total number of pages in apHash */ - unsigned int nHash; /* Number of slots in apHash[] */ - PgHdr1 **apHash; /* Hash table for fast lookup by key */ - - unsigned int iMaxKey; /* Largest key seen since xTruncate() */ -}; - -/* -** Each cache entry is represented by an instance of the following -** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated -** directly before this structure in memory (see the PGHDR1_TO_PAGE() -** macro below). +** This file implements that page cache. */ -struct PgHdr1 { - unsigned int iKey; /* Key value (page number) */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; /* -** Free slots in the allocator used to divide up the buffer provided using -** the SQLITE_CONFIG_PAGECACHE mechanism. +** A complete page cache is an instance of this structure. */ -struct PgFreeslot { - PgFreeslot *pNext; /* Next free slot */ +struct PCache { + PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ + PgHdr *pSynced; /* Last synced page in dirty page list */ + int nRef; /* Number of referenced pages */ + int nMax; /* Configured cache size */ + int szPage; /* Size of every page in this cache */ + int szExtra; /* Size of extra space for each page */ + int bPurgeable; /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ + void *pStress; /* Argument to xStress */ + sqlite3_pcache *pCache; /* Pluggable cache module */ + PgHdr *pPage1; /* Reference to page 1 */ }; /* -** Global data used by this cache. +** Some of the assert() macros in this code are too expensive to run +** even during normal debugging. Use them only rarely on long-running +** tests. Enable the expensive asserts using the +** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option. */ -static SQLITE_WSD struct PCacheGlobal { - sqlite3_mutex *mutex; /* static mutex MUTEX_STATIC_LRU */ - - int nMaxPage; /* Sum of nMaxPage for purgeable caches */ - int nMinPage; /* Sum of nMinPage for purgeable caches */ - int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ - - /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */ - int szSlot; /* Size of each free slot */ - int nSlot; /* The number of pcache slots */ - int nFreeSlot; /* Number of unused pcache slots */ - int nReserve; /* Try to keep nFreeSlot above this */ - void *pStart, *pEnd; /* Bounds of pagecache malloc range */ - PgFreeslot *pFree; /* Free page blocks */ - int isInit; /* True if initialized */ -} pcache1_g; +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT +# define expensive_assert(X) assert(X) +#else +# define expensive_assert(X) +#endif -/* -** All code in this file should access the global structure above via the -** alias "pcache1". This ensures that the WSD emulation is used when -** compiling for systems that do not support real WSD. -*/ -#define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) +/********************************** Linked List Management ********************/ +#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) /* -** When a PgHdr1 structure is allocated, the associated PCache1.szPage -** bytes of data are located directly before it in memory (i.e. the total -** size of the allocation is sizeof(PgHdr1)+PCache1.szPage byte). The -** PGHDR1_TO_PAGE() macro takes a pointer to a PgHdr1 structure as -** an argument and returns a pointer to the associated block of szPage -** bytes. The PAGE_TO_PGHDR1() macro does the opposite: its argument is -** a pointer to a block of szPage bytes of data and the return value is -** a pointer to the associated PgHdr1 structure. +** Check that the pCache->pSynced variable is set correctly. If it +** is not, either fail an assert or return zero. Otherwise, return +** non-zero. This is only used in debugging builds, as follows: ** -** assert( PGHDR1_TO_PAGE(PAGE_TO_PGHDR1(pCache, X))==X ); +** expensive_assert( pcacheCheckSynced(pCache) ); */ -#define PGHDR1_TO_PAGE(p) (void*)(((char*)p) - p->pCache->szPage) -#define PAGE_TO_PGHDR1(c, p) (PgHdr1*)(((char*)p) + c->szPage) +static int pcacheCheckSynced(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ + assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); + } + return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); +} +#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ /* -** Macros to enter and leave the global LRU mutex. +** Remove page pPage from the list of dirty pages. */ -#define pcache1EnterMutex() sqlite3_mutex_enter(pcache1.mutex) -#define pcache1LeaveMutex() sqlite3_mutex_leave(pcache1.mutex) +static void pcacheRemoveFromDirtyList(PgHdr *pPage){ + PCache *p = pPage->pCache; -/******************************************************************************/ -/******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); -/* -** This function is called during initialization if a static buffer is -** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE -** verb to sqlite3_config(). Parameter pBuf points to an allocation large -** enough to contain 'n' buffers of 'sz' bytes each. -*/ -SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ - if( pcache1.isInit ){ - PgFreeslot *p; - sz = ROUNDDOWN8(sz); - pcache1.szSlot = sz; - pcache1.nSlot = pcache1.nFreeSlot = n; - pcache1.nReserve = n>90 ? 10 : (n/10 + 1); - pcache1.pStart = pBuf; - pcache1.pFree = 0; - while( n-- ){ - p = (PgFreeslot*)pBuf; - p->pNext = pcache1.pFree; - pcache1.pFree = p; - pBuf = (void*)&((char*)pBuf)[sz]; + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + PgHdr *pSynced = pPage->pDirtyPrev; + while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ + pSynced = pSynced->pDirtyPrev; } - pcache1.pEnd = pBuf; + p->pSynced = pSynced; } -} -/* -** Malloc function used within this file to allocate space from the buffer -** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no -** such buffer exists or there is no space left in it, this function falls -** back to sqlite3Malloc(). -*/ -static void *pcache1Alloc(int nByte){ - void *p; - assert( sqlite3_mutex_held(pcache1.mutex) ); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); - if( nByte<=pcache1.szSlot && pcache1.pFree ){ - assert( pcache1.isInit ); - p = (PgHdr1 *)pcache1.pFree; - pcache1.pFree = pcache1.pFree->pNext; - pcache1.nFreeSlot--; - assert( pcache1.nFreeSlot>=0 ); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; }else{ - - /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the - ** global pcache mutex and unlock the pager-cache object pCache. This is - ** so that if the attempt to allocate a new buffer causes the the - ** configured soft-heap-limit to be breached, it will be possible to - ** reclaim memory from this pager-cache. - */ - pcache1LeaveMutex(); - p = sqlite3Malloc(nByte); - pcache1EnterMutex(); - if( p ){ - int sz = sqlite3MallocSize(p); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); - } - sqlite3MemdebugSetType(p, MEMTYPE_PCACHE); + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; } - return p; -} - -/* -** Free an allocated buffer obtained from pcache1Alloc(). -*/ -static void pcache1Free(void *p){ - assert( sqlite3_mutex_held(pcache1.mutex) ); - if( p==0 ) return; - if( p>=pcache1.pStart && ppNext = pcache1.pFree; - pcache1.pFree = pSlot; - pcache1.nFreeSlot++; - assert( pcache1.nFreeSlot<=pcache1.nSlot ); + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; }else{ - int iSize; - assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - iSize = sqlite3MallocSize(p); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); - sqlite3_free(p); + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; } + pPage->pDirtyNext = 0; + pPage->pDirtyPrev = 0; + + expensive_assert( pcacheCheckSynced(p) ); } -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* -** Return the size of a pcache allocation +** Add page pPage to the head of the dirty list (PCache1.pDirty is set to +** pPage). */ -static int pcache1MemSize(void *p){ - assert( sqlite3_mutex_held(pcache1.mutex) ); - if( p>=pcache1.pStart && ppCache; + + assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); + + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; } + p->pDirty = pPage; + if( !p->pDirtyTail ){ + p->pDirtyTail = pPage; + } + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + p->pSynced = pPage; + } + expensive_assert( pcacheCheckSynced(p) ); } -#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ /* -** Allocate a new page object initially associated with cache pCache. +** Wrapper around the pluggable caches xUnpin method. If the cache is +** being used for an in-memory database, this function is a no-op. */ -static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ - int nByte = sizeof(PgHdr1) + pCache->szPage; - void *pPg = pcache1Alloc(nByte); - PgHdr1 *p; - if( pPg ){ - p = PAGE_TO_PGHDR1(pCache, pPg); - if( pCache->bPurgeable ){ - pcache1.nCurrentPage++; +static void pcacheUnpin(PgHdr *p){ + PCache *pCache = p->pCache; + if( pCache->bPurgeable ){ + if( p->pgno==1 ){ + pCache->pPage1 = 0; } - }else{ - p = 0; + sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 0); } - return p; } -/* -** Free a page object allocated by pcache1AllocPage(). +/*************************************************** General Interfaces ****** ** -** The pointer is allowed to be NULL, which is prudent. But it turns out -** that the current implementation happens to never call this routine -** with a NULL pointer, so we mark the NULL test with ALWAYS(). +** Initialize and shutdown the page cache subsystem. Neither of these +** functions are threadsafe. */ -static void pcache1FreePage(PgHdr1 *p){ - if( ALWAYS(p) ){ - if( p->pCache->bPurgeable ){ - pcache1.nCurrentPage--; - } - pcache1Free(PGHDR1_TO_PAGE(p)); +SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ + if( sqlite3GlobalConfig.pcache.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ + sqlite3PCacheSetDefault(); + } + return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); +} +SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ + if( sqlite3GlobalConfig.pcache.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ + sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); } } /* -** Malloc function used by SQLite to obtain space from the buffer configured -** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer -** exists, this function falls back to sqlite3Malloc(). +** Return the size in bytes of a PCache object. */ -SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ - void *p; - pcache1EnterMutex(); - p = pcache1Alloc(sz); - pcache1LeaveMutex(); - return p; -} +SQLITE_PRIVATE int sqlite3PcacheSize(void){ return sizeof(PCache); } /* -** Free an allocated buffer obtained from sqlite3PageMalloc(). +** Create a new PCache object. Storage space to hold the object +** has already been allocated and is passed in as the p pointer. +** The caller discovers how much space needs to be allocated by +** calling sqlite3PcacheSize(). */ -SQLITE_PRIVATE void sqlite3PageFree(void *p){ - pcache1EnterMutex(); - pcache1Free(p); - pcache1LeaveMutex(); +SQLITE_PRIVATE void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *p /* Preallocated space for the PCache */ +){ + memset(p, 0, sizeof(PCache)); + p->szPage = szPage; + p->szExtra = szExtra; + p->bPurgeable = bPurgeable; + p->xStress = xStress; + p->pStress = pStress; + p->nMax = 100; } - /* -** Return true if it desirable to avoid allocating a new page cache -** entry. -** -** If memory was allocated specifically to the page cache using -** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then -** it is desirable to avoid allocating a new page cache entry because -** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient -** for all page cache needs and we should not need to spill the -** allocation onto the heap. -** -** Or, the heap is used for all page cache memory put the heap is -** under memory pressure, then again it is desirable to avoid -** allocating a new page cache entry in order to avoid stressing -** the heap even further. +** Change the page size for PCache object. The caller must ensure that there +** are no outstanding page references when this function is called. */ -static int pcache1UnderMemoryPressure(PCache1 *pCache){ - assert( sqlite3_mutex_held(pcache1.mutex) ); - if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){ - return pcache1.nFreeSlotnRef==0 && pCache->pDirty==0 ); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); + pCache->pCache = 0; + pCache->pPage1 = 0; } + pCache->szPage = szPage; } -/******************************************************************************/ -/******** General Implementation Functions ************************************/ - /* -** This function is used to resize the hash table used by the cache passed -** as the first argument. -** -** The global mutex must be held when this function is called. +** Try to obtain a page from the cache. */ -static int pcache1ResizeHash(PCache1 *p){ - PgHdr1 **apNew; - unsigned int nNew; - unsigned int i; - - assert( sqlite3_mutex_held(pcache1.mutex) ); - - nNew = p->nHash*2; - if( nNew<256 ){ - nNew = 256; +SQLITE_PRIVATE int sqlite3PcacheFetch( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + int createFlag, /* If true, create page if it does not exist already */ + PgHdr **ppPage /* Write the page here */ +){ + PgHdr *pPage = 0; + int eCreate; + + assert( pCache!=0 ); + assert( createFlag==1 || createFlag==0 ); + assert( pgno>0 ); + + /* If the pluggable cache (sqlite3_pcache*) has not been allocated, + ** allocate it now. + */ + if( !pCache->pCache && createFlag ){ + sqlite3_pcache *p; + int nByte; + nByte = pCache->szPage + pCache->szExtra + sizeof(PgHdr); + p = sqlite3GlobalConfig.pcache.xCreate(nByte, pCache->bPurgeable); + if( !p ){ + return SQLITE_NOMEM; + } + sqlite3GlobalConfig.pcache.xCachesize(p, pCache->nMax); + pCache->pCache = p; } - pcache1LeaveMutex(); - if( p->nHash ){ sqlite3BeginBenignMalloc(); } - apNew = (PgHdr1 **)sqlite3_malloc(sizeof(PgHdr1 *)*nNew); - if( p->nHash ){ sqlite3EndBenignMalloc(); } - pcache1EnterMutex(); - if( apNew ){ - memset(apNew, 0, sizeof(PgHdr1 *)*nNew); - for(i=0; inHash; i++){ - PgHdr1 *pPage; - PgHdr1 *pNext = p->apHash[i]; - while( (pPage = pNext)!=0 ){ - unsigned int h = pPage->iKey % nNew; - pNext = pPage->pNext; - pPage->pNext = apNew[h]; - apNew[h] = pPage; + eCreate = createFlag * (1 + (!pCache->bPurgeable || !pCache->pDirty)); + if( pCache->pCache ){ + pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, eCreate); + } + + if( !pPage && eCreate==1 ){ + PgHdr *pPg; + + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + expensive_assert( pcacheCheckSynced(pCache) ); + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; +#ifdef SQLITE_LOG_CACHE_SPILL + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + pCache->nMax); +#endif + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; } } - sqlite3_free(p->apHash); - p->apHash = apNew; - p->nHash = nNew; + + pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, 2); } - return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); + if( pPage ){ + if( !pPage->pData ){ + memset(pPage, 0, sizeof(PgHdr)); + pPage->pData = (void *)&pPage[1]; + pPage->pExtra = (void*)&((char *)pPage->pData)[pCache->szPage]; + memset(pPage->pExtra, 0, pCache->szExtra); + pPage->pCache = pCache; + pPage->pgno = pgno; + } + assert( pPage->pCache==pCache ); + assert( pPage->pgno==pgno ); + assert( pPage->pData==(void *)&pPage[1] ); + assert( pPage->pExtra==(void *)&((char *)&pPage[1])[pCache->szPage] ); + + if( 0==pPage->nRef ){ + pCache->nRef++; + } + pPage->nRef++; + if( pgno==1 ){ + pCache->pPage1 = pPage; + } + } + *ppPage = pPage; + return (pPage==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; } /* -** This function is used internally to remove the page pPage from the -** global LRU list, if is part of it. If pPage is not part of the global -** LRU list, then this function is a no-op. -** -** The global mutex must be held when this function is called. +** Decrement the reference count on a page. If the page is clean and the +** reference count drops to 0, then it is made elible for recycling. */ -static void pcache1PinPage(PgHdr1 *pPage){ - assert( sqlite3_mutex_held(pcache1.mutex) ); - if( pPage && (pPage->pLruNext || pPage==pcache1.pLruTail) ){ - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - } - if( pcache1.pLruHead==pPage ){ - pcache1.pLruHead = pPage->pLruNext; - } - if( pcache1.pLruTail==pPage ){ - pcache1.pLruTail = pPage->pLruPrev; +SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr *p){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ){ + PCache *pCache = p->pCache; + pCache->nRef--; + if( (p->flags&PGHDR_DIRTY)==0 ){ + pcacheUnpin(p); + }else{ + /* Move the page to the head of the dirty list. */ + pcacheRemoveFromDirtyList(p); + pcacheAddToDirtyList(p); } - pPage->pLruNext = 0; - pPage->pLruPrev = 0; - pPage->pCache->nRecyclable--; } } - /* -** Remove the page supplied as an argument from the hash table -** (PCache1.apHash structure) that it is currently stored in. -** -** The global mutex must be held when this function is called. +** Increase the reference count of a supplied page by 1. */ -static void pcache1RemoveFromHash(PgHdr1 *pPage){ - unsigned int h; - PCache1 *pCache = pPage->pCache; - PgHdr1 **pp; - - h = pPage->iKey % pCache->nHash; - for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); - *pp = (*pp)->pNext; +SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ + assert(p->nRef>0); + p->nRef++; +} - pCache->nPage--; +/* +** Drop a page from the cache. There must be exactly one reference to the +** page. This function deletes that reference, so after it returns the +** page pointed to by p is invalid. +*/ +SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ + PCache *pCache; + assert( p->nRef==1 ); + if( p->flags&PGHDR_DIRTY ){ + pcacheRemoveFromDirtyList(p); + } + pCache = p->pCache; + pCache->nRef--; + if( p->pgno==1 ){ + pCache->pPage1 = 0; + } + sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 1); } /* -** If there are currently more than pcache.nMaxPage pages allocated, try -** to recycle pages to reduce the number allocated to pcache.nMaxPage. +** Make sure the page is marked as dirty. If it isn't dirty already, +** make it so. */ -static void pcache1EnforceMaxPage(void){ - assert( sqlite3_mutex_held(pcache1.mutex) ); - while( pcache1.nCurrentPage>pcache1.nMaxPage && pcache1.pLruTail ){ - PgHdr1 *p = pcache1.pLruTail; - pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); +SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ + p->flags &= ~PGHDR_DONT_WRITE; + assert( p->nRef>0 ); + if( 0==(p->flags & PGHDR_DIRTY) ){ + p->flags |= PGHDR_DIRTY; + pcacheAddToDirtyList( p); } } /* -** Discard all pages from cache pCache with a page number (key value) -** greater than or equal to iLimit. Any pinned pages that meet this -** criteria are unpinned before they are discarded. -** -** The global mutex must be held when this function is called. +** Make sure the page is marked as clean. If it isn't clean already, +** make it so. */ -static void pcache1TruncateUnsafe( - PCache1 *pCache, - unsigned int iLimit -){ - TESTONLY( unsigned int nPage = 0; ) /* Used to assert pCache->nPage is correct */ - unsigned int h; - assert( sqlite3_mutex_held(pcache1.mutex) ); - for(h=0; hnHash; h++){ - PgHdr1 **pp = &pCache->apHash[h]; - PgHdr1 *pPage; - while( (pPage = *pp)!=0 ){ - if( pPage->iKey>=iLimit ){ - pCache->nPage--; - *pp = pPage->pNext; - pcache1PinPage(pPage); - pcache1FreePage(pPage); - }else{ - pp = &pPage->pNext; - TESTONLY( nPage++; ) - } +SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ + if( (p->flags & PGHDR_DIRTY) ){ + pcacheRemoveFromDirtyList(p); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); + if( p->nRef==0 ){ + pcacheUnpin(p); } } - assert( pCache->nPage==nPage ); } -/******************************************************************************/ -/******** sqlite3_pcache Methods **********************************************/ +/* +** Make every page in the cache clean. +*/ +SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache *pCache){ + PgHdr *p; + while( (p = pCache->pDirty)!=0 ){ + sqlite3PcacheMakeClean(p); + } +} /* -** Implementation of the sqlite3_pcache.xInit method. +** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ -static int pcache1Init(void *NotUsed){ - UNUSED_PARAMETER(NotUsed); - assert( pcache1.isInit==0 ); - memset(&pcache1, 0, sizeof(pcache1)); - if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); +SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~PGHDR_NEED_SYNC; } - pcache1.isInit = 1; - return SQLITE_OK; + pCache->pSynced = pCache->pDirtyTail; } /* -** Implementation of the sqlite3_pcache.xShutdown method. -** Note that the static mutex allocated in xInit does -** not need to be freed. +** Change the page number of page p to newPgno. */ -static void pcache1Shutdown(void *NotUsed){ - UNUSED_PARAMETER(NotUsed); - assert( pcache1.isInit!=0 ); - memset(&pcache1, 0, sizeof(pcache1)); +SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ + PCache *pCache = p->pCache; + assert( p->nRef>0 ); + assert( newPgno>0 ); + sqlite3GlobalConfig.pcache.xRekey(pCache->pCache, p, p->pgno, newPgno); + p->pgno = newPgno; + if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ + pcacheRemoveFromDirtyList(p); + pcacheAddToDirtyList(p); + } } /* -** Implementation of the sqlite3_pcache.xCreate method. +** Drop every cache entry whose page number is greater than "pgno". The +** caller must ensure that there are no outstanding references to any pages +** other than page 1 with a page number greater than pgno. ** -** Allocate a new cache. +** If there is a reference to page 1 and the pgno parameter passed to this +** function is 0, then the data area associated with page 1 is zeroed, but +** the page object is not dropped. */ -static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ - PCache1 *pCache; - - pCache = (PCache1 *)sqlite3_malloc(sizeof(PCache1)); - if( pCache ){ - memset(pCache, 0, sizeof(PCache1)); - pCache->szPage = szPage; - pCache->bPurgeable = (bPurgeable ? 1 : 0); - if( bPurgeable ){ - pCache->nMin = 10; - pcache1EnterMutex(); - pcache1.nMinPage += pCache->nMin; - pcache1LeaveMutex(); +SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ + if( pCache->pCache ){ + PgHdr *p; + PgHdr *pNext; + for(p=pCache->pDirty; p; p=pNext){ + pNext = p->pDirtyNext; + /* This routine never gets call with a positive pgno except right + ** after sqlite3PcacheCleanAll(). So if there are dirty pages, + ** it must be that pgno==0. + */ + assert( p->pgno>0 ); + if( ALWAYS(p->pgno>pgno) ){ + assert( p->flags&PGHDR_DIRTY ); + sqlite3PcacheMakeClean(p); + } + } + if( pgno==0 && pCache->pPage1 ){ + memset(pCache->pPage1->pData, 0, pCache->szPage); + pgno = 1; } + sqlite3GlobalConfig.pcache.xTruncate(pCache->pCache, pgno+1); } - return (sqlite3_pcache *)pCache; } /* -** Implementation of the sqlite3_pcache.xCachesize method. -** -** Configure the cache_size limit for a cache. +** Close a cache. */ -static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ - PCache1 *pCache = (PCache1 *)p; - if( pCache->bPurgeable ){ - pcache1EnterMutex(); - pcache1.nMaxPage += (nMax - pCache->nMax); - pCache->nMax = nMax; - pcache1EnforceMaxPage(); - pcache1LeaveMutex(); +SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); } } -/* -** Implementation of the sqlite3_pcache.xPagecount method. +/* +** Discard the contents of the cache. */ -static int pcache1Pagecount(sqlite3_pcache *p){ - int n; - pcache1EnterMutex(); - n = ((PCache1 *)p)->nPage; - pcache1LeaveMutex(); - return n; +SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ + sqlite3PcacheTruncate(pCache, 0); } /* -** Implementation of the sqlite3_pcache.xFetch method. -** -** Fetch a page by key value. -** -** Whether or not a new page may be allocated by this function depends on -** the value of the createFlag argument. 0 means do not allocate a new -** page. 1 means allocate a new page if space is easily available. 2 -** means to try really hard to allocate a new page. -** -** For a non-purgeable cache (a cache used as the storage for an in-memory -** database) there is really no difference between createFlag 1 and 2. So -** the calling function (pcache.c) will never have a createFlag of 1 on -** a non-purgable cache. -** -** There are three different approaches to obtaining space for a page, -** depending on the value of parameter createFlag (which may be 0, 1 or 2). -** -** 1. Regardless of the value of createFlag, the cache is searched for a -** copy of the requested page. If one is found, it is returned. -** -** 2. If createFlag==0 and the page is not already in the cache, NULL is -** returned. -** -** 3. If createFlag is 1, and the page is not already in the cache, then -** return NULL (do not allocate a new page) if any of the following -** conditions are true: -** -** (a) the number of pages pinned by the cache is greater than -** PCache1.nMax, or -** -** (b) the number of pages pinned by the cache is greater than -** the sum of nMax for all purgeable caches, less the sum of -** nMin for all other purgeable caches, or -** -** 4. If none of the first three conditions apply and the cache is marked -** as purgeable, and if one of the following is true: -** -** (a) The number of pages allocated for the cache is already -** PCache1.nMax, or -** -** (b) The number of pages allocated for all purgeable caches is -** already equal to or greater than the sum of nMax for all -** purgeable caches, -** -** (c) The system is under memory pressure and wants to avoid -** unnecessary pages cache entry allocations -** -** then attempt to recycle a page from the LRU list. If it is the right -** size, return the recycled buffer. Otherwise, free the buffer and -** proceed to step 5. -** -** 5. Otherwise, allocate and return a new page buffer. +** Merge two lists of pages connected by pDirty and in pgno order. +** Do not both fixing the pDirtyPrev pointers. */ -static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ - unsigned int nPinned; - PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage = 0; - - assert( pCache->bPurgeable || createFlag!=1 ); - pcache1EnterMutex(); - if( createFlag==1 ) sqlite3BeginBenignMalloc(); - - /* Search the hash table for an existing entry. */ - if( pCache->nHash>0 ){ - unsigned int h = iKey % pCache->nHash; - for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); - } - - if( pPage || createFlag==0 ){ - pcache1PinPage(pPage); - goto fetch_out; - } - - /* Step 3 of header comment. */ - nPinned = pCache->nPage - pCache->nRecyclable; - if( createFlag==1 && ( - nPinned>=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage) - || nPinned>=(pCache->nMax * 9 / 10) - || pcache1UnderMemoryPressure(pCache) - )){ - goto fetch_out; - } - - if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ - goto fetch_out; - } - - /* Step 4. Try to recycle a page buffer if appropriate. */ - if( pCache->bPurgeable && pcache1.pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pcache1.nCurrentPage>=pcache1.nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ - pPage = pcache1.pLruTail; - pcache1RemoveFromHash(pPage); - pcache1PinPage(pPage); - if( pPage->pCache->szPage!=pCache->szPage ){ - pcache1FreePage(pPage); - pPage = 0; +static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ + PgHdr result, *pTail; + pTail = &result; + while( pA && pB ){ + if( pA->pgnopgno ){ + pTail->pDirty = pA; + pTail = pA; + pA = pA->pDirty; }else{ - pcache1.nCurrentPage -= (pPage->pCache->bPurgeable - pCache->bPurgeable); + pTail->pDirty = pB; + pTail = pB; + pB = pB->pDirty; } } - - /* Step 5. If a usable page buffer has still not been found, - ** attempt to allocate a new one. - */ - if( !pPage ){ - pPage = pcache1AllocPage(pCache); - } - - if( pPage ){ - unsigned int h = iKey % pCache->nHash; - pCache->nPage++; - pPage->iKey = iKey; - pPage->pNext = pCache->apHash[h]; - pPage->pCache = pCache; - pPage->pLruPrev = 0; - pPage->pLruNext = 0; - *(void **)(PGHDR1_TO_PAGE(pPage)) = 0; - pCache->apHash[h] = pPage; - } - -fetch_out: - if( pPage && iKey>pCache->iMaxKey ){ - pCache->iMaxKey = iKey; + if( pA ){ + pTail->pDirty = pA; + }else if( pB ){ + pTail->pDirty = pB; + }else{ + pTail->pDirty = 0; } - if( createFlag==1 ) sqlite3EndBenignMalloc(); - pcache1LeaveMutex(); - return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); + return result.pDirty; } - /* -** Implementation of the sqlite3_pcache.xUnpin method. +** Sort the list of pages in accending order by pgno. Pages are +** connected by pDirty pointers. The pDirtyPrev pointers are +** corrupted by this sort. ** -** Mark a page as unpinned (eligible for asynchronous recycling). +** Since there cannot be more than 2^31 distinct pages in a database, +** there cannot be more than 31 buckets required by the merge sorter. +** One extra bucket is added to catch overflow in case something +** ever changes to make the previous sentence incorrect. */ -static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){ - PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg); - - assert( pPage->pCache==pCache ); - pcache1EnterMutex(); - - /* It is an error to call this function if the page is already - ** part of the global LRU list. - */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pcache1.pLruHead!=pPage && pcache1.pLruTail!=pPage ); - - if( reuseUnlikely || pcache1.nCurrentPage>pcache1.nMaxPage ){ - pcache1RemoveFromHash(pPage); - pcache1FreePage(pPage); - }else{ - /* Add the page to the global LRU list. Normally, the page is added to - ** the head of the list (last page to be recycled). However, if the - ** reuseUnlikely flag passed to this function is true, the page is added - ** to the tail of the list (first page to be recycled). - */ - if( pcache1.pLruHead ){ - pcache1.pLruHead->pLruPrev = pPage; - pPage->pLruNext = pcache1.pLruHead; - pcache1.pLruHead = pPage; - }else{ - pcache1.pLruTail = pPage; - pcache1.pLruHead = pPage; +#define N_SORT_BUCKET 32 +static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ + PgHdr *a[N_SORT_BUCKET], *p; + int i; + memset(a, 0, sizeof(a)); + while( pIn ){ + p = pIn; + pIn = p->pDirty; + p->pDirty = 0; + for(i=0; ALWAYS(inRecyclable++; } - - pcache1LeaveMutex(); + p = a[0]; + for(i=1; iiKey==iOld ); - assert( pPage->pCache==pCache ); - - pcache1EnterMutex(); - - h = iOld%pCache->nHash; - pp = &pCache->apHash[h]; - while( (*pp)!=pPage ){ - pp = &(*pp)->pNext; +SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->pDirty = p->pDirtyNext; } - *pp = pPage->pNext; + return pcacheSortDirtyList(pCache->pDirty); +} - h = iNew%pCache->nHash; - pPage->iKey = iNew; - pPage->pNext = pCache->apHash[h]; - pCache->apHash[h] = pPage; - if( iNew>pCache->iMaxKey ){ - pCache->iMaxKey = iNew; - } - - pcache1LeaveMutex(); +/* +** Return the total number of referenced pages held by the cache. +*/ +SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ + return pCache->nRef; } /* -** Implementation of the sqlite3_pcache.xTruncate method. -** -** Discard all unpinned pages in the cache with a page number equal to -** or greater than parameter iLimit. Any pinned pages with a page number -** equal to or greater than iLimit are implicitly unpinned. +** Return the number of references to the page supplied as an argument. */ -static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ - PCache1 *pCache = (PCache1 *)p; - pcache1EnterMutex(); - if( iLimit<=pCache->iMaxKey ){ - pcache1TruncateUnsafe(pCache, iLimit); - pCache->iMaxKey = iLimit-1; - } - pcache1LeaveMutex(); +SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ + return p->nRef; } -/* -** Implementation of the sqlite3_pcache.xDestroy method. -** -** Destroy a cache allocated using pcache1Create(). +/* +** Return the total number of pages in the cache. */ -static void pcache1Destroy(sqlite3_pcache *p){ - PCache1 *pCache = (PCache1 *)p; - assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); - pcache1EnterMutex(); - pcache1TruncateUnsafe(pCache, 0); - pcache1.nMaxPage -= pCache->nMax; - pcache1.nMinPage -= pCache->nMin; - pcache1EnforceMaxPage(); - pcache1LeaveMutex(); - sqlite3_free(pCache->apHash); - sqlite3_free(pCache); +SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ + int nPage = 0; + if( pCache->pCache ){ + nPage = sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache); + } + return nPage; } +#ifdef SQLITE_TEST /* -** This function is called during initialization (sqlite3_initialize()) to -** install the default pluggable cache module, assuming the user has not -** already provided an alternative. +** Get the suggested cache-size value. */ -SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ - static const sqlite3_pcache_methods defaultMethods = { - 0, /* pArg */ - pcache1Init, /* xInit */ - pcache1Shutdown, /* xShutdown */ - pcache1Create, /* xCreate */ - pcache1Cachesize, /* xCachesize */ - pcache1Pagecount, /* xPagecount */ - pcache1Fetch, /* xFetch */ - pcache1Unpin, /* xUnpin */ - pcache1Rekey, /* xRekey */ - pcache1Truncate, /* xTruncate */ - pcache1Destroy /* xDestroy */ - }; - sqlite3_config(SQLITE_CONFIG_PCACHE, &defaultMethods); +SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *pCache){ + return pCache->nMax; } +#endif -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* -** This function is called to free superfluous dynamically allocated memory -** held by the pager system. Memory in use by any SQLite pager allocated -** by the current thread may be sqlite3_free()ed. -** -** nReq is the number of bytes of memory required. Once this much has -** been released, the function returns. The return value is the total number -** of bytes of memory released. +** Set the suggested cache-size value. */ -SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ - int nFree = 0; - if( pcache1.pStart==0 ){ - PgHdr1 *p; - pcache1EnterMutex(); - while( (nReq<0 || nFreenMax = mxPage; + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache.xCachesize(pCache->pCache, mxPage); } - return nFree; } -#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ -#ifdef SQLITE_TEST +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* -** This function is used by test procedures to inspect the internal state -** of the global cache. +** For all dirty pages currently in the cache, invoke the specified +** callback. This is only used if the SQLITE_CHECK_PAGES macro is +** defined. */ -SQLITE_PRIVATE void sqlite3PcacheStats( - int *pnCurrent, /* OUT: Total number of pages cached */ - int *pnMax, /* OUT: Global maximum cache size */ - int *pnMin, /* OUT: Sum of PCache1.nMin for purgeable caches */ - int *pnRecyclable /* OUT: Total number of pages available for recycling */ -){ - PgHdr1 *p; - int nRecyclable = 0; - for(p=pcache1.pLruHead; p; p=p->pLruNext){ - nRecyclable++; +SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)){ + PgHdr *pDirty; + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext){ + xIter(pDirty); } - *pnCurrent = pcache1.nCurrentPage; - *pnMax = pcache1.nMaxPage; - *pnMin = pcache1.nMinPage; - *pnRecyclable = nRecyclable; } #endif -/************** End of pcache1.c *********************************************/ -/************** Begin file rowset.c ******************************************/ +/************** End of pcache.c **********************************************/ +/************** Begin file pcache1.c *****************************************/ /* -** 2008 December 3 +** 2008 November 05 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -34177,48095 +35290,46270 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** ************************************************************************* ** -** This module implements an object we call a "RowSet". -** -** The RowSet object is a collection of rowids. Rowids -** are inserted into the RowSet in an arbitrary order. Inserts -** can be intermixed with tests to see if a given rowid has been -** previously inserted into the RowSet. -** -** After all inserts are finished, it is possible to extract the -** elements of the RowSet in sorted order. Once this extraction -** process has started, no new elements may be inserted. -** -** Hence, the primitive operations for a RowSet are: -** -** CREATE -** INSERT -** TEST -** SMALLEST -** DESTROY +** This file implements the default page cache implementation (the +** sqlite3_pcache interface). It also contains part of the implementation +** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. +** If the default page cache implementation is overriden, then neither of +** these two features are available. +*/ + + +typedef struct PCache1 PCache1; +typedef struct PgHdr1 PgHdr1; +typedef struct PgFreeslot PgFreeslot; +typedef struct PGroup PGroup; + +/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set +** of one or more PCaches that are able to recycle each others unpinned +** pages when they are under memory pressure. A PGroup is an instance of +** the following object. ** -** The CREATE and DESTROY primitives are the constructor and destructor, -** obviously. The INSERT primitive adds a new element to the RowSet. -** TEST checks to see if an element is already in the RowSet. SMALLEST -** extracts the least value from the RowSet. +** This page cache implementation works in one of two modes: ** -** The INSERT primitive might allocate additional memory. Memory is -** allocated in chunks so most INSERTs do no allocation. There is an -** upper bound on the size of allocated memory. No memory is freed -** until DESTROY. +** (1) Every PCache is the sole member of its own PGroup. There is +** one PGroup per PCache. ** -** The TEST primitive includes a "batch" number. The TEST primitive -** will only see elements that were inserted before the last change -** in the batch number. In other words, if an INSERT occurs between -** two TESTs where the TESTs have the same batch nubmer, then the -** value added by the INSERT will not be visible to the second TEST. -** The initial batch number is zero, so if the very first TEST contains -** a non-zero batch number, it will see all prior INSERTs. +** (2) There is a single global PGroup that all PCaches are a member +** of. ** -** No INSERTs may occurs after a SMALLEST. An assertion will fail if -** that is attempted. +** Mode 1 uses more memory (since PCache instances are not able to rob +** unused pages from other PCaches) but it also operates without a mutex, +** and is therefore often faster. Mode 2 requires a mutex in order to be +** threadsafe, but is able recycle pages more efficient. ** -** The cost of an INSERT is roughly constant. (Sometime new memory -** has to be allocated on an INSERT.) The cost of a TEST with a new -** batch number is O(NlogN) where N is the number of elements in the RowSet. -** The cost of a TEST using the same batch number is O(logN). The cost -** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST -** primitives are constant time. The cost of DESTROY is O(N). +** For mode (1), PGroup.mutex is NULL. For mode (2) there is only a single +** PGroup which is the pcache1.grp global variable and its mutex is +** SQLITE_MUTEX_STATIC_LRU. +*/ +struct PGroup { + sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */ + int nMaxPage; /* Sum of nMax for purgeable caches */ + int nMinPage; /* Sum of nMin for purgeable caches */ + int mxPinned; /* nMaxpage + 10 - nMinPage */ + int nCurrentPage; /* Number of purgeable pages allocated */ + PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ +}; + +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. ** -** There is an added cost of O(N) when switching between TEST and -** SMALLEST primitives. +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. */ +struct PCache1 { + /* Cache configuration parameters. Page size (szPage) and the purgeable + ** flag (bPurgeable) are set when the cache is created. nMax may be + ** modified at any time by a call to the pcache1CacheSize() method. + ** The PGroup mutex must be held when accessing nMax. + */ + PGroup *pGroup; /* PGroup this cache belongs to */ + int szPage; /* Size of allocated pages in bytes */ + int bPurgeable; /* True if cache is purgeable */ + unsigned int nMin; /* Minimum number of pages reserved */ + unsigned int nMax; /* Configured "cache_size" value */ + unsigned int n90pct; /* nMax*9/10 */ + /* Hash table of all pages. The following variables may only be accessed + ** when the accessor is holding the PGroup mutex. + */ + unsigned int nRecyclable; /* Number of pages in the LRU list */ + unsigned int nPage; /* Total number of pages in apHash */ + unsigned int nHash; /* Number of slots in apHash[] */ + PgHdr1 **apHash; /* Hash table for fast lookup by key */ + + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ +}; /* -** Target size for allocation chunks. +** Each cache entry is represented by an instance of the following +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly before this structure in memory (see the PGHDR1_TO_PAGE() +** macro below). */ -#define ROWSET_ALLOCATION_SIZE 1024 +struct PgHdr1 { + unsigned int iKey; /* Key value (page number) */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; /* -** The number of rowset entries per allocation chunk. +** Free slots in the allocator used to divide up the buffer provided using +** the SQLITE_CONFIG_PAGECACHE mechanism. */ -#define ROWSET_ENTRY_PER_CHUNK \ - ((ROWSET_ALLOCATION_SIZE-8)/sizeof(struct RowSetEntry)) +struct PgFreeslot { + PgFreeslot *pNext; /* Next free slot */ +}; /* -** Each entry in a RowSet is an instance of the following object. +** Global data used by this cache. */ -struct RowSetEntry { - i64 v; /* ROWID value for this entry */ - struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ - struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ -}; +static SQLITE_WSD struct PCacheGlobal { + PGroup grp; /* The global PGroup for mode (2) */ + + /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The + ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all + ** fixed at sqlite3_initialize() time and do not require mutex protection. + ** The nFreeSlot and pFree values do require mutex protection. + */ + int isInit; /* True if initialized */ + int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ + void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + /* Above requires no mutex. Use mutex below for variable that follow. */ + sqlite3_mutex *mutex; /* Mutex for accessing the following: */ + int nFreeSlot; /* Number of unused pcache slots */ + PgFreeslot *pFree; /* Free page blocks */ + /* The following value requires a mutex to change. We skip the mutex on + ** reading because (1) most platforms read a 32-bit integer atomically and + ** (2) even if an incorrect value is read, no great harm is done since this + ** is really just an optimization. */ + int bUnderPressure; /* True if low on PAGECACHE memory */ +} pcache1_g; /* -** RowSetEntry objects are allocated in large chunks (instances of the -** following structure) to reduce memory allocation overhead. The -** chunks are kept on a linked list so that they can be deallocated -** when the RowSet is destroyed. +** All code in this file should access the global structure above via the +** alias "pcache1". This ensures that the WSD emulation is used when +** compiling for systems that do not support real WSD. */ -struct RowSetChunk { - struct RowSetChunk *pNextChunk; /* Next chunk on list of them all */ - struct RowSetEntry aEntry[ROWSET_ENTRY_PER_CHUNK]; /* Allocated entries */ -}; +#define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) /* -** A RowSet in an instance of the following structure. +** When a PgHdr1 structure is allocated, the associated PCache1.szPage +** bytes of data are located directly before it in memory (i.e. the total +** size of the allocation is sizeof(PgHdr1)+PCache1.szPage byte). The +** PGHDR1_TO_PAGE() macro takes a pointer to a PgHdr1 structure as +** an argument and returns a pointer to the associated block of szPage +** bytes. The PAGE_TO_PGHDR1() macro does the opposite: its argument is +** a pointer to a block of szPage bytes of data and the return value is +** a pointer to the associated PgHdr1 structure. ** -** A typedef of this structure if found in sqliteInt.h. +** assert( PGHDR1_TO_PAGE(PAGE_TO_PGHDR1(pCache, X))==X ); */ -struct RowSet { - struct RowSetChunk *pChunk; /* List of all chunk allocations */ - sqlite3 *db; /* The database connection */ - struct RowSetEntry *pEntry; /* List of entries using pRight */ - struct RowSetEntry *pLast; /* Last entry on the pEntry list */ - struct RowSetEntry *pFresh; /* Source of new entry objects */ - struct RowSetEntry *pTree; /* Binary tree of entries */ - u16 nFresh; /* Number of objects on pFresh */ - u8 isSorted; /* True if pEntry is sorted */ - u8 iBatch; /* Current insert batch */ -}; +#define PGHDR1_TO_PAGE(p) (void*)(((char*)p) - p->pCache->szPage) +#define PAGE_TO_PGHDR1(c, p) (PgHdr1*)(((char*)p) + c->szPage) /* -** Turn bulk memory into a RowSet object. N bytes of memory -** are available at pSpace. The db pointer is used as a memory context -** for any subsequent allocations that need to occur. -** Return a pointer to the new RowSet object. -** -** It must be the case that N is sufficient to make a Rowset. If not -** an assertion fault occurs. -** -** If N is larger than the minimum, use the surplus as an initial -** allocation of entries available to be filled. +** Macros to enter and leave the PCache LRU mutex. */ -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ - RowSet *p; - assert( N >= ROUND8(sizeof(*p)) ); - p = pSpace; - p->pChunk = 0; - p->db = db; - p->pEntry = 0; - p->pLast = 0; - p->pTree = 0; - p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); - p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); - p->isSorted = 1; - p->iBatch = 0; - return p; -} +#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) + +/******************************************************************************/ +/******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ /* -** Deallocate all chunks from a RowSet. This frees all memory that -** the RowSet has allocated over its lifetime. This routine is -** the destructor for the RowSet. +** This function is called during initialization if a static buffer is +** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE +** verb to sqlite3_config(). Parameter pBuf points to an allocation large +** enough to contain 'n' buffers of 'sz' bytes each. +** +** This routine is called from sqlite3_initialize() and so it is guaranteed +** to be serialized already. There is no need for further mutexing. */ -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ - struct RowSetChunk *pChunk, *pNextChunk; - for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ - pNextChunk = pChunk->pNextChunk; - sqlite3DbFree(p->db, pChunk); +SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ + if( pcache1.isInit ){ + PgFreeslot *p; + sz = ROUNDDOWN8(sz); + pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); + pcache1.pStart = pBuf; + pcache1.pFree = 0; + pcache1.bUnderPressure = 0; + while( n-- ){ + p = (PgFreeslot*)pBuf; + p->pNext = pcache1.pFree; + pcache1.pFree = p; + pBuf = (void*)&((char*)pBuf)[sz]; + } + pcache1.pEnd = pBuf; } - p->pChunk = 0; - p->nFresh = 0; - p->pEntry = 0; - p->pLast = 0; - p->pTree = 0; - p->isSorted = 1; } /* -** Insert a new value into a RowSet. +** Malloc function used within this file to allocate space from the buffer +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls +** back to sqlite3Malloc(). ** -** The mallocFailed flag of the database connection is set if a -** memory allocation fails. +** Multiple threads can run this routine at the same time. Global variables +** in pcache1 need to be protected via mutex. */ -SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ - struct RowSetEntry *pEntry; /* The new entry */ - struct RowSetEntry *pLast; /* The last prior entry */ - assert( p!=0 ); - if( p->nFresh==0 ){ - struct RowSetChunk *pNew; - pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); - if( pNew==0 ){ - return; +static void *pcache1Alloc(int nByte){ + void *p = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + if( nByte<=pcache1.szSlot ){ + sqlite3_mutex_enter(pcache1.mutex); + p = (PgHdr1 *)pcache1.pFree; + if( p ){ + pcache1.pFree = pcache1.pFree->pNext; + pcache1.nFreeSlot--; + pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); } - pNew->pNextChunk = p->pChunk; - p->pChunk = pNew; - p->pFresh = pNew->aEntry; - p->nFresh = ROWSET_ENTRY_PER_CHUNK; + sqlite3_mutex_leave(pcache1.mutex); } - pEntry = p->pFresh++; - p->nFresh--; - pEntry->v = rowid; - pEntry->pRight = 0; - pLast = p->pLast; - if( pLast ){ - if( p->isSorted && rowid<=pLast->v ){ - p->isSorted = 0; + if( p==0 ){ + /* Memory is not available in the SQLITE_CONFIG_PAGECACHE pool. Get + ** it from sqlite3Malloc instead. + */ + p = sqlite3Malloc(nByte); + if( p ){ + int sz = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + sqlite3_mutex_leave(pcache1.mutex); } - pLast->pRight = pEntry; - }else{ - assert( p->pEntry==0 ); /* Fires if INSERT after SMALLEST */ - p->pEntry = pEntry; + sqlite3MemdebugSetType(p, MEMTYPE_PCACHE); } - p->pLast = pEntry; + return p; } /* -** Merge two lists of RowSetEntry objects. Remove duplicates. -** -** The input lists are connected via pRight pointers and are -** assumed to each already be in sorted order. +** Free an allocated buffer obtained from pcache1Alloc(). */ -static struct RowSetEntry *rowSetMerge( - struct RowSetEntry *pA, /* First sorted list to be merged */ - struct RowSetEntry *pB /* Second sorted list to be merged */ -){ - struct RowSetEntry head; - struct RowSetEntry *pTail; - - pTail = &head; - while( pA && pB ){ - assert( pA->pRight==0 || pA->v<=pA->pRight->v ); - assert( pB->pRight==0 || pB->v<=pB->pRight->v ); - if( pA->vv ){ - pTail->pRight = pA; - pA = pA->pRight; - pTail = pTail->pRight; - }else if( pB->vv ){ - pTail->pRight = pB; - pB = pB->pRight; - pTail = pTail->pRight; - }else{ - pA = pA->pRight; - } - } - if( pA ){ - assert( pA->pRight==0 || pA->v<=pA->pRight->v ); - pTail->pRight = pA; +static void pcache1Free(void *p){ + if( p==0 ) return; + if( p>=pcache1.pStart && ppNext = pcache1.pFree; + pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + pcache1.bUnderPressure = pcache1.nFreeSlotpRight==0 || pB->v<=pB->pRight->v ); - pTail->pRight = pB; + int iSize; + assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); + sqlite3_mutex_leave(pcache1.mutex); + sqlite3_free(p); } - return head.pRight; } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* -** Sort all elements on the pEntry list of the RowSet into ascending order. -*/ -static void rowSetSort(RowSet *p){ - unsigned int i; - struct RowSetEntry *pEntry; - struct RowSetEntry *aBucket[40]; - - assert( p->isSorted==0 ); - memset(aBucket, 0, sizeof(aBucket)); - while( p->pEntry ){ - pEntry = p->pEntry; - p->pEntry = pEntry->pRight; - pEntry->pRight = 0; - for(i=0; aBucket[i]; i++){ - pEntry = rowSetMerge(aBucket[i], pEntry); - aBucket[i] = 0; - } - aBucket[i] = pEntry; - } - pEntry = 0; - for(i=0; i=pcache1.pStart && ppEntry = pEntry; - p->pLast = 0; - p->isSorted = 1; } - +#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ /* -** The input, pIn, is a binary tree (or subtree) of RowSetEntry objects. -** Convert this tree into a linked list connected by the pRight pointers -** and return pointers to the first and last elements of the new list. +** Allocate a new page object initially associated with cache pCache. */ -static void rowSetTreeToList( - struct RowSetEntry *pIn, /* Root of the input tree */ - struct RowSetEntry **ppFirst, /* Write head of the output list here */ - struct RowSetEntry **ppLast /* Write tail of the output list here */ -){ - assert( pIn!=0 ); - if( pIn->pLeft ){ - struct RowSetEntry *p; - rowSetTreeToList(pIn->pLeft, ppFirst, &p); - p->pRight = pIn; - }else{ - *ppFirst = pIn; - } - if( pIn->pRight ){ - rowSetTreeToList(pIn->pRight, &pIn->pRight, ppLast); +static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ + int nByte = sizeof(PgHdr1) + pCache->szPage; + void *pPg = pcache1Alloc(nByte); + PgHdr1 *p; + if( pPg ){ + p = PAGE_TO_PGHDR1(pCache, pPg); + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } }else{ - *ppLast = pIn; + p = 0; } - assert( (*ppLast)->pRight==0 ); + return p; } - /* -** Convert a sorted list of elements (connected by pRight) into a binary -** tree with depth of iDepth. A depth of 1 means the tree contains a single -** node taken from the head of *ppList. A depth of 2 means a tree with -** three nodes. And so forth. -** -** Use as many entries from the input list as required and update the -** *ppList to point to the unused elements of the list. If the input -** list contains too few elements, then construct an incomplete tree -** and leave *ppList set to NULL. +** Free a page object allocated by pcache1AllocPage(). ** -** Return a pointer to the root of the constructed binary tree. +** The pointer is allowed to be NULL, which is prudent. But it turns out +** that the current implementation happens to never call this routine +** with a NULL pointer, so we mark the NULL test with ALWAYS(). */ -static struct RowSetEntry *rowSetNDeepTree( - struct RowSetEntry **ppList, - int iDepth -){ - struct RowSetEntry *p; /* Root of the new tree */ - struct RowSetEntry *pLeft; /* Left subtree */ - if( *ppList==0 ){ - return 0; - } - if( iDepth==1 ){ - p = *ppList; - *ppList = p->pRight; - p->pLeft = p->pRight = 0; - return p; - } - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ - return pLeft; +static void pcache1FreePage(PgHdr1 *p){ + if( ALWAYS(p) ){ + PCache1 *pCache = p->pCache; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; + } + pcache1Free(PGHDR1_TO_PAGE(p)); } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); - return p; } /* -** Convert a sorted list of elements into a binary tree. Make the tree -** as deep as it needs to be in order to contain the entire list. +** Malloc function used by SQLite to obtain space from the buffer configured +** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer +** exists, this function falls back to sqlite3Malloc(). */ -static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ - int iDepth; /* Depth of the tree so far */ - struct RowSetEntry *p; /* Current tree root */ - struct RowSetEntry *pLeft; /* Left subtree */ - - assert( pList!=0 ); - p = pList; - pList = p->pRight; - p->pLeft = p->pRight = 0; - for(iDepth=1; pList; iDepth++){ - pLeft = p; - p = pList; - pList = p->pRight; - p->pLeft = pLeft; - p->pRight = rowSetNDeepTree(&pList, iDepth); - } - return p; +SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ + return pcache1Alloc(sz); } /* -** Convert the list in p->pEntry into a sorted list if it is not -** sorted already. If there is a binary tree on p->pTree, then -** convert it into a list too and merge it into the p->pEntry list. +** Free an allocated buffer obtained from sqlite3PageMalloc(). */ -static void rowSetToList(RowSet *p){ - if( !p->isSorted ){ - rowSetSort(p); - } - if( p->pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(p->pTree, &pHead, &pTail); - p->pTree = 0; - p->pEntry = rowSetMerge(p->pEntry, pHead); - } +SQLITE_PRIVATE void sqlite3PageFree(void *p){ + pcache1Free(p); } + /* -** Extract the smallest element from the RowSet. -** Write the element into *pRowid. Return 1 on success. Return -** 0 if the RowSet is already empty. +** Return true if it desirable to avoid allocating a new page cache +** entry. ** -** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. +** If memory was allocated specifically to the page cache using +** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then +** it is desirable to avoid allocating a new page cache entry because +** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient +** for all page cache needs and we should not need to spill the +** allocation onto the heap. +** +** Or, the heap is used for all page cache memory put the heap is +** under memory pressure, then again it is desirable to avoid +** allocating a new page cache entry in order to avoid stressing +** the heap even further. */ -SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ - rowSetToList(p); - if( p->pEntry ){ - *pRowid = p->pEntry->v; - p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ - sqlite3RowSetClear(p); - } - return 1; +static int pcache1UnderMemoryPressure(PCache1 *pCache){ + if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){ + return pcache1.bUnderPressure; }else{ - return 0; + return sqlite3HeapNearlyFull(); } } +/******************************************************************************/ +/******** General Implementation Functions ************************************/ + /* -** Check to see if element iRowid was inserted into the the rowset as -** part of any insert batch prior to iBatch. Return 1 or 0. +** This function is used to resize the hash table used by the cache passed +** as the first argument. +** +** The PCache mutex must be held when this function is called. */ -SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 iRowid){ - struct RowSetEntry *p; - if( iBatch!=pRowSet->iBatch ){ - if( pRowSet->pEntry ){ - rowSetToList(pRowSet); - pRowSet->pTree = rowSetListToTree(pRowSet->pEntry); - pRowSet->pEntry = 0; - pRowSet->pLast = 0; - } - pRowSet->iBatch = iBatch; +static int pcache1ResizeHash(PCache1 *p){ + PgHdr1 **apNew; + unsigned int nNew; + unsigned int i; + + assert( sqlite3_mutex_held(p->pGroup->mutex) ); + + nNew = p->nHash*2; + if( nNew<256 ){ + nNew = 256; } - p = pRowSet->pTree; - while( p ){ - if( p->vpRight; - }else if( p->v>iRowid ){ - p = p->pLeft; - }else{ - return 1; + + pcache1LeaveMutex(p->pGroup); + if( p->nHash ){ sqlite3BeginBenignMalloc(); } + apNew = (PgHdr1 **)sqlite3_malloc(sizeof(PgHdr1 *)*nNew); + if( p->nHash ){ sqlite3EndBenignMalloc(); } + pcache1EnterMutex(p->pGroup); + if( apNew ){ + memset(apNew, 0, sizeof(PgHdr1 *)*nNew); + for(i=0; inHash; i++){ + PgHdr1 *pPage; + PgHdr1 *pNext = p->apHash[i]; + while( (pPage = pNext)!=0 ){ + unsigned int h = pPage->iKey % nNew; + pNext = pPage->pNext; + pPage->pNext = apNew[h]; + apNew[h] = pPage; + } } + sqlite3_free(p->apHash); + p->apHash = apNew; + p->nHash = nNew; } - return 0; + + return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); } -/************** End of rowset.c **********************************************/ -/************** Begin file pager.c *******************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** This function is used internally to remove the page pPage from the +** PGroup LRU list, if is part of it. If pPage is not part of the PGroup +** LRU list, then this function is a no-op. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** The PGroup mutex must be held when this function is called. ** -************************************************************************* -** This is the implementation of the page cache subsystem or "pager". -** -** The pager is used to access a database disk file. It implements -** atomic commit and rollback through the use of a journal file that -** is separate from the database file. The pager also implements file -** locking to prevent two processes from writing the same database -** file simultaneously, or one process from reading the database while -** another is writing. +** If pPage is NULL then this routine is a no-op. */ -#ifndef SQLITE_OMIT_DISKIO -/************** Include wal.h in the middle of pager.c ***********************/ -/************** Begin file wal.h *********************************************/ +static void pcache1PinPage(PgHdr1 *pPage){ + PCache1 *pCache; + PGroup *pGroup; + + if( pPage==0 ) return; + pCache = pPage->pCache; + pGroup = pCache->pGroup; + assert( sqlite3_mutex_held(pGroup->mutex) ); + if( pPage->pLruNext || pPage==pGroup->pLruTail ){ + if( pPage->pLruPrev ){ + pPage->pLruPrev->pLruNext = pPage->pLruNext; + } + if( pPage->pLruNext ){ + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + } + if( pGroup->pLruHead==pPage ){ + pGroup->pLruHead = pPage->pLruNext; + } + if( pGroup->pLruTail==pPage ){ + pGroup->pLruTail = pPage->pLruPrev; + } + pPage->pLruNext = 0; + pPage->pLruPrev = 0; + pPage->pCache->nRecyclable--; + } +} + + /* -** 2010 February 1 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Remove the page supplied as an argument from the hash table +** (PCache1.apHash structure) that it is currently stored in. ** -************************************************************************* -** This header file defines the interface to the write-ahead logging -** system. Refer to the comments below and the header comment attached to -** the implementation of each function in log.c for further details. +** The PGroup mutex must be held when this function is called. */ +static void pcache1RemoveFromHash(PgHdr1 *pPage){ + unsigned int h; + PCache1 *pCache = pPage->pCache; + PgHdr1 **pp; -#ifndef _WAL_H_ -#define _WAL_H_ - - -#ifdef SQLITE_OMIT_WAL -# define sqlite3WalOpen(x,y,z) 0 -# define sqlite3WalClose(w,x,y,z) 0 -# define sqlite3WalBeginReadTransaction(y,z) 0 -# define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 -# define sqlite3WalDbsize(y) 0 -# define sqlite3WalBeginWriteTransaction(y) 0 -# define sqlite3WalEndWriteTransaction(x) 0 -# define sqlite3WalUndo(x,y,z) 0 -# define sqlite3WalSavepoint(y,z) -# define sqlite3WalSavepointUndo(y,z) 0 -# define sqlite3WalFrames(u,v,w,x,y,z) 0 -# define sqlite3WalCheckpoint(u,v,w,x) 0 -# define sqlite3WalCallback(z) 0 -# define sqlite3WalExclusiveMode(y,z) 0 -# define sqlite3WalHeapMemory(z) 0 -#else + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + h = pPage->iKey % pCache->nHash; + for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); + *pp = (*pp)->pNext; -#define WAL_SAVEPOINT_NDATA 4 + pCache->nPage--; +} -/* Connection to a write-ahead log (WAL) file. -** There is one object of this type for each pager. +/* +** If there are currently more than nMaxPage pages allocated, try +** to recycle pages to reduce the number allocated to nMaxPage. */ -typedef struct Wal Wal; - -/* Open and close a connection to a write-ahead log. */ -SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, int, Wal**); -SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); +static void pcache1EnforceMaxPage(PGroup *pGroup){ + assert( sqlite3_mutex_held(pGroup->mutex) ); + while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ + PgHdr1 *p = pGroup->pLruTail; + assert( p->pCache->pGroup==pGroup ); + pcache1PinPage(p); + pcache1RemoveFromHash(p); + pcache1FreePage(p); + } +} -/* Used by readers to open (lock) and close (unlock) a snapshot. A -** snapshot is like a read-transaction. It is the state of the database -** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and -** preserves the current state even if the other threads or processes -** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the -** transaction and releases the lock. +/* +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this +** criteria are unpinned before they are discarded. +** +** The PCache mutex must be held when this function is called. */ -SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *); -SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal); - -/* Read a page from the write-ahead log, if it is present. */ -SQLITE_PRIVATE int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); - -/* If the WAL is not empty, return the size of the database. */ -SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal); - -/* Obtain or release the WRITER lock. */ -SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal); -SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal); - -/* Undo any frames written (but not committed) to the log */ -SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); - -/* Return an integer that records the current (uncommitted) write -** position in the WAL */ -SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); - -/* Move the write position of the WAL back to iFrame. Called in -** response to a ROLLBACK TO command. */ -SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); - -/* Write a frame or frames to the log. */ -SQLITE_PRIVATE int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); +static void pcache1TruncateUnsafe( + PCache1 *pCache, /* The cache to truncate */ + unsigned int iLimit /* Drop pages with this pgno or larger */ +){ + TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h; + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + for(h=0; hnHash; h++){ + PgHdr1 **pp = &pCache->apHash[h]; + PgHdr1 *pPage; + while( (pPage = *pp)!=0 ){ + if( pPage->iKey>=iLimit ){ + pCache->nPage--; + *pp = pPage->pNext; + pcache1PinPage(pPage); + pcache1FreePage(pPage); + }else{ + pp = &pPage->pNext; + TESTONLY( nPage++; ) + } + } + } + assert( pCache->nPage==nPage ); +} -/* Copy pages from the log to the database file */ -SQLITE_PRIVATE int sqlite3WalCheckpoint( - Wal *pWal, /* Write-ahead log connection */ - int sync_flags, /* Flags to sync db file with (or 0) */ - int nBuf, /* Size of buffer nBuf */ - u8 *zBuf /* Temporary buffer to use */ -); +/******************************************************************************/ +/******** sqlite3_pcache Methods **********************************************/ -/* Return the value to pass to a sqlite3_wal_hook callback, the -** number of frames in the WAL at the point of the last commit since -** sqlite3WalCallback() was called. If no commits have occurred since -** the last call, then return 0. +/* +** Implementation of the sqlite3_pcache.xInit method. */ -SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal); +static int pcache1Init(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit==0 ); + memset(&pcache1, 0, sizeof(pcache1)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + } + pcache1.grp.mxPinned = 10; + pcache1.isInit = 1; + return SQLITE_OK; +} -/* Tell the wal layer that an EXCLUSIVE lock has been obtained (or released) -** by the pager layer on the database file. +/* +** Implementation of the sqlite3_pcache.xShutdown method. +** Note that the static mutex allocated in xInit does +** not need to be freed. */ -SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); +static void pcache1Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit!=0 ); + memset(&pcache1, 0, sizeof(pcache1)); +} -/* Return true if the argument is non-NULL and the WAL module is using -** heap-memory for the wal-index. Otherwise, if the argument is NULL or the -** WAL module is using shared-memory, return false. +/* +** Implementation of the sqlite3_pcache.xCreate method. +** +** Allocate a new cache. */ -SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); +static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ + PCache1 *pCache; /* The newly created page cache */ + PGroup *pGroup; /* The group the new page cache will belong to */ + int sz; /* Bytes of memory required to allocate the new cache */ -#endif /* ifndef SQLITE_OMIT_WAL */ -#endif /* _WAL_H_ */ + /* + ** The seperateCache variable is true if each PCache has its own private + ** PGroup. In other words, separateCache is true for mode (1) where no + ** mutexing is required. + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Always use a unified cache in single-threaded applications + ** + ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) + ** use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 + const int separateCache = 0; +#else + int separateCache = sqlite3GlobalConfig.bCoreMutex>0; +#endif -/************** End of wal.h *************************************************/ -/************** Continuing where we left off in pager.c **********************/ + sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; + pCache = (PCache1 *)sqlite3_malloc(sz); + if( pCache ){ + memset(pCache, 0, sz); + if( separateCache ){ + pGroup = (PGroup*)&pCache[1]; + pGroup->mxPinned = 10; + }else{ + pGroup = &pcache1.grp; + } + pCache->pGroup = pGroup; + pCache->szPage = szPage; + pCache->bPurgeable = (bPurgeable ? 1 : 0); + if( bPurgeable ){ + pCache->nMin = 10; + pcache1EnterMutex(pGroup); + pGroup->nMinPage += pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pcache1LeaveMutex(pGroup); + } + } + return (sqlite3_pcache *)pCache; +} +/* +** Implementation of the sqlite3_pcache.xCachesize method. +** +** Configure the cache_size limit for a cache. +*/ +static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ + PCache1 *pCache = (PCache1 *)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + pcache1EnterMutex(pGroup); + pGroup->nMaxPage += (nMax - pCache->nMax); + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->nMax = nMax; + pCache->n90pct = pCache->nMax*9/10; + pcache1EnforceMaxPage(pGroup); + pcache1LeaveMutex(pGroup); + } +} -/******************* NOTES ON THE DESIGN OF THE PAGER ************************ +/* +** Implementation of the sqlite3_pcache.xPagecount method. +*/ +static int pcache1Pagecount(sqlite3_pcache *p){ + int n; + PCache1 *pCache = (PCache1*)p; + pcache1EnterMutex(pCache->pGroup); + n = pCache->nPage; + pcache1LeaveMutex(pCache->pGroup); + return n; +} + +/* +** Implementation of the sqlite3_pcache.xFetch method. ** -** This comment block describes invariants that hold when using a rollback -** journal. These invariants do not apply for journal_mode=WAL, -** journal_mode=MEMORY, or journal_mode=OFF. +** Fetch a page by key value. ** -** Within this comment block, a page is deemed to have been synced -** automatically as soon as it is written when PRAGMA synchronous=OFF. -** Otherwise, the page is not synced until the xSync method of the VFS -** is called successfully on the file containing the page. +** Whether or not a new page may be allocated by this function depends on +** the value of the createFlag argument. 0 means do not allocate a new +** page. 1 means allocate a new page if space is easily available. 2 +** means to try really hard to allocate a new page. ** -** Definition: A page of the database file is said to be "overwriteable" if -** one or more of the following are true about the page: -** -** (a) The original content of the page as it was at the beginning of -** the transaction has been written into the rollback journal and -** synced. -** -** (b) The page was a freelist leaf page at the start of the transaction. -** -** (c) The page number is greater than the largest page that existed in -** the database file at the start of the transaction. -** -** (1) A page of the database file is never overwritten unless one of the -** following are true: -** -** (a) The page and all other pages on the same sector are overwriteable. -** -** (b) The atomic page write optimization is enabled, and the entire -** transaction other than the update of the transaction sequence -** number consists of a single page change. -** -** (2) The content of a page written into the rollback journal exactly matches -** both the content in the database when the rollback journal was written -** and the content in the database at the beginning of the current -** transaction. -** -** (3) Writes to the database file are an integer multiple of the page size -** in length and are aligned on a page boundary. -** -** (4) Reads from the database file are either aligned on a page boundary and -** an integer multiple of the page size in length or are taken from the -** first 100 bytes of the database file. -** -** (5) All writes to the database file are synced prior to the rollback journal -** being deleted, truncated, or zeroed. -** -** (6) If a master journal file is used, then all writes to the database file -** are synced prior to the master journal being deleted. -** -** Definition: Two databases (or the same database at two points it time) -** are said to be "logically equivalent" if they give the same answer to -** all queries. Note in particular the the content of freelist leaf -** pages can be changed arbitarily without effecting the logical equivalence -** of the database. -** -** (7) At any time, if any subset, including the empty set and the total set, -** of the unsynced changes to a rollback journal are removed and the -** journal is rolled back, the resulting database file will be logical -** equivalent to the database file at the beginning of the transaction. -** -** (8) When a transaction is rolled back, the xTruncate method of the VFS -** is called to restore the database file to the same size it was at -** the beginning of the transaction. (In some VFSes, the xTruncate -** method is a no-op, but that does not change the fact the SQLite will -** invoke it.) -** -** (9) Whenever the database file is modified, at least one bit in the range -** of bytes from 24 through 39 inclusive will be changed prior to releasing -** the EXCLUSIVE lock, thus signaling other connections on the same -** database to flush their caches. +** For a non-purgeable cache (a cache used as the storage for an in-memory +** database) there is really no difference between createFlag 1 and 2. So +** the calling function (pcache.c) will never have a createFlag of 1 on +** a non-purgable cache. ** -** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less -** than one billion transactions. +** There are three different approaches to obtaining space for a page, +** depending on the value of parameter createFlag (which may be 0, 1 or 2). ** -** (11) A database file is well-formed at the beginning and at the conclusion -** of every transaction. +** 1. Regardless of the value of createFlag, the cache is searched for a +** copy of the requested page. If one is found, it is returned. ** -** (12) An EXCLUSIVE lock is held on the database file when writing to -** the database file. +** 2. If createFlag==0 and the page is not already in the cache, NULL is +** returned. ** -** (13) A SHARED lock is held on the database file while reading any -** content out of the database file. +** 3. If createFlag is 1, and the page is not already in the cache, then +** return NULL (do not allocate a new page) if any of the following +** conditions are true: ** -******************************************************************************/ - -/* -** Macros for troubleshooting. Normally turned off -*/ -#if 0 -int sqlite3PagerTrace=1; /* True to enable tracing */ -#define sqlite3DebugPrintf printf -#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; } -#else -#define PAGERTRACE(X) -#endif - -/* -** The following two macros are used within the PAGERTRACE() macros above -** to print out file-descriptors. +** (a) the number of pages pinned by the cache is greater than +** PCache1.nMax, or ** -** PAGERID() takes a pointer to a Pager struct as its argument. The -** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file -** struct as its argument. -*/ -#define PAGERID(p) ((int)(p->fd)) -#define FILEHANDLEID(fd) ((int)fd) - -/* -** The Pager.eState variable stores the current 'state' of a pager. A -** pager may be in any one of the seven states shown in the following -** state diagram. +** (b) the number of pages pinned by the cache is greater than +** the sum of nMax for all purgeable caches, less the sum of +** nMin for all other purgeable caches, or ** -** OPEN <------+------+ -** | | | -** V | | -** +---------> READER-------+ | -** | | | -** | V | -** |<-------WRITER_LOCKED------> ERROR -** | | ^ -** | V | -** |<------WRITER_CACHEMOD-------->| -** | | | -** | V | -** |<-------WRITER_DBMOD---------->| -** | | | -** | V | -** +<------WRITER_FINISHED-------->+ -** -** -** List of state transitions and the C [function] that performs each: -** -** OPEN -> READER [sqlite3PagerSharedLock] -** READER -> OPEN [pager_unlock] -** -** READER -> WRITER_LOCKED [sqlite3PagerBegin] -** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] -** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] -** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] -** WRITER_*** -> READER [pager_end_transaction] -** -** WRITER_*** -> ERROR [pager_error] -** ERROR -> OPEN [pager_unlock] -** -** -** OPEN: -** -** The pager starts up in this state. Nothing is guaranteed in this -** state - the file may or may not be locked and the database size is -** unknown. The database may not be read or written. -** -** * No read or write transaction is active. -** * Any lock, or no lock at all, may be held on the database file. -** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. -** -** READER: -** -** In this state all the requirements for reading the database in -** rollback (non-WAL) mode are met. Unless the pager is (or recently -** was) in exclusive-locking mode, a user-level read transaction is -** open. The database size is known in this state. -** -** A connection running with locking_mode=normal enters this state when -** it opens a read-transaction on the database and returns to state -** OPEN after the read-transaction is completed. However a connection -** running in locking_mode=exclusive (including temp databases) remains in -** this state even after the read-transaction is closed. The only way -** a locking_mode=exclusive connection can transition from READER to OPEN -** is via the ERROR state (see below). -** -** * A read transaction may be active (but a write-transaction cannot). -** * A SHARED or greater lock is held on the database file. -** * The dbSize variable may be trusted (even if a user-level read -** transaction is not active). The dbOrigSize and dbFileSize variables -** may not be trusted at this point. -** * If the database is a WAL database, then the WAL connection is open. -** * Even if a read-transaction is not open, it is guaranteed that -** there is no hot-journal in the file-system. -** -** WRITER_LOCKED: -** -** The pager moves to this state from READER when a write-transaction -** is first opened on the database. In WRITER_LOCKED state, all locks -** required to start a write-transaction are held, but no actual -** modifications to the cache or database have taken place. -** -** In rollback mode, a RESERVED or (if the transaction was opened with -** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when -** moving to this state, but the journal file is not written to or opened -** to in this state. If the transaction is committed or rolled back while -** in WRITER_LOCKED state, all that is required is to unlock the database -** file. -** -** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. -** If the connection is running with locking_mode=exclusive, an attempt -** is made to obtain an EXCLUSIVE lock on the database file. -** -** * A write transaction is active. -** * If the connection is open in rollback-mode, a RESERVED or greater -** lock is held on the database file. -** * If the connection is open in WAL-mode, a WAL write transaction -** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully -** called). -** * The dbSize, dbOrigSize and dbFileSize variables are all valid. -** * The contents of the pager cache have not been modified. -** * The journal file may or may not be open. -** * Nothing (not even the first header) has been written to the journal. -** -** WRITER_CACHEMOD: -** -** A pager moves from WRITER_LOCKED state to this state when a page is -** first modified by the upper layer. In rollback mode the journal file -** is opened (if it is not already open) and a header written to the -** start of it. The database file on disk has not been modified. -** -** * A write transaction is active. -** * A RESERVED or greater lock is held on the database file. -** * The journal file is open and the first header has been written -** to it, but the header has not been synced to disk. -** * The contents of the page cache have been modified. +** 4. If none of the first three conditions apply and the cache is marked +** as purgeable, and if one of the following is true: ** -** WRITER_DBMOD: +** (a) The number of pages allocated for the cache is already +** PCache1.nMax, or ** -** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state -** when it modifies the contents of the database file. WAL connections -** never enter this state (since they do not modify the database file, -** just the log file). +** (b) The number of pages allocated for all purgeable caches is +** already equal to or greater than the sum of nMax for all +** purgeable caches, ** -** * A write transaction is active. -** * An EXCLUSIVE or greater lock is held on the database file. -** * The journal file is open and the first header has been written -** and synced to disk. -** * The contents of the page cache have been modified (and possibly -** written to disk). +** (c) The system is under memory pressure and wants to avoid +** unnecessary pages cache entry allocations ** -** WRITER_FINISHED: +** then attempt to recycle a page from the LRU list. If it is the right +** size, return the recycled buffer. Otherwise, free the buffer and +** proceed to step 5. ** -** It is not possible for a WAL connection to enter this state. +** 5. Otherwise, allocate and return a new page buffer. +*/ +static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ + int nPinned; + PCache1 *pCache = (PCache1 *)p; + PGroup *pGroup; + PgHdr1 *pPage = 0; + + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + pcache1EnterMutex(pGroup = pCache->pGroup); + + /* Step 1: Search the hash table for an existing entry. */ + if( pCache->nHash>0 ){ + unsigned int h = iKey % pCache->nHash; + for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); + } + + /* Step 2: Abort if no existing page is found and createFlag is 0 */ + if( pPage || createFlag==0 ){ + pcache1PinPage(pPage); + goto fetch_out; + } + + /* The pGroup local variable will normally be initialized by the + ** pcache1EnterMutex() macro above. But if SQLITE_MUTEX_OMIT is defined, + ** then pcache1EnterMutex() is a no-op, so we have to initialize the + ** local variable here. Delaying the initialization of pGroup is an + ** optimization: The common case is to exit the module before reaching + ** this point. + */ +#ifdef SQLITE_MUTEX_OMIT + pGroup = pCache->pGroup; +#endif + + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + nPinned = pCache->nPage - pCache->nRecyclable; + assert( nPinned>=0 ); + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=(int)pCache->n90pct + || pcache1UnderMemoryPressure(pCache) + )){ + goto fetch_out; + } + + if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ + goto fetch_out; + } + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable && pGroup->pLruTail && ( + (pCache->nPage+1>=pCache->nMax) + || pGroup->nCurrentPage>=pGroup->nMaxPage + || pcache1UnderMemoryPressure(pCache) + )){ + PCache1 *pOtherCache; + pPage = pGroup->pLruTail; + pcache1RemoveFromHash(pPage); + pcache1PinPage(pPage); + if( (pOtherCache = pPage->pCache)->szPage!=pCache->szPage ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nCurrentPage -= + (pOtherCache->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + if( createFlag==1 ) sqlite3BeginBenignMalloc(); + pcache1LeaveMutex(pGroup); + pPage = pcache1AllocPage(pCache); + pcache1EnterMutex(pGroup); + if( createFlag==1 ) sqlite3EndBenignMalloc(); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruPrev = 0; + pPage->pLruNext = 0; + *(void **)(PGHDR1_TO_PAGE(pPage)) = 0; + pCache->apHash[h] = pPage; + } + +fetch_out: + if( pPage && iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + pcache1LeaveMutex(pGroup); + return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); +} + + +/* +** Implementation of the sqlite3_pcache.xUnpin method. ** -** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD -** state after the entire transaction has been successfully written into the -** database file. In this state the transaction may be committed simply -** by finalizing the journal file. Once in WRITER_FINISHED state, it is -** not possible to modify the database further. At this point, the upper -** layer must either commit or rollback the transaction. +** Mark a page as unpinned (eligible for asynchronous recycling). +*/ +static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg); + PGroup *pGroup = pCache->pGroup; + + assert( pPage->pCache==pCache ); + pcache1EnterMutex(pGroup); + + /* It is an error to call this function if the page is already + ** part of the PGroup LRU list. + */ + assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); + + if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ + pcache1RemoveFromHash(pPage); + pcache1FreePage(pPage); + }else{ + /* Add the page to the PGroup LRU list. */ + if( pGroup->pLruHead ){ + pGroup->pLruHead->pLruPrev = pPage; + pPage->pLruNext = pGroup->pLruHead; + pGroup->pLruHead = pPage; + }else{ + pGroup->pLruTail = pPage; + pGroup->pLruHead = pPage; + } + pCache->nRecyclable++; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xRekey method. +*/ +static void pcache1Rekey( + sqlite3_pcache *p, + void *pPg, + unsigned int iOld, + unsigned int iNew +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = PAGE_TO_PGHDR1(pCache, pPg); + PgHdr1 **pp; + unsigned int h; + assert( pPage->iKey==iOld ); + assert( pPage->pCache==pCache ); + + pcache1EnterMutex(pCache->pGroup); + + h = iOld%pCache->nHash; + pp = &pCache->apHash[h]; + while( (*pp)!=pPage ){ + pp = &(*pp)->pNext; + } + *pp = pPage->pNext; + + h = iNew%pCache->nHash; + pPage->iKey = iNew; + pPage->pNext = pCache->apHash[h]; + pCache->apHash[h] = pPage; + if( iNew>pCache->iMaxKey ){ + pCache->iMaxKey = iNew; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xTruncate method. ** -** * A write transaction is active. -** * An EXCLUSIVE or greater lock is held on the database file. -** * All writing and syncing of journal and database data has finished. -** If no error occured, all that remains is to finalize the journal to -** commit the transaction. If an error did occur, the caller will need -** to rollback the transaction. +** Discard all unpinned pages in the cache with a page number equal to +** or greater than parameter iLimit. Any pinned pages with a page number +** equal to or greater than iLimit are implicitly unpinned. +*/ +static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ + PCache1 *pCache = (PCache1 *)p; + pcache1EnterMutex(pCache->pGroup); + if( iLimit<=pCache->iMaxKey ){ + pcache1TruncateUnsafe(pCache, iLimit); + pCache->iMaxKey = iLimit-1; + } + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xDestroy method. ** -** ERROR: +** Destroy a cache allocated using pcache1Create(). +*/ +static void pcache1Destroy(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1 *)p; + PGroup *pGroup = pCache->pGroup; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); + pcache1EnterMutex(pGroup); + pcache1TruncateUnsafe(pCache, 0); + pGroup->nMaxPage -= pCache->nMax; + pGroup->nMinPage -= pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pcache1EnforceMaxPage(pGroup); + pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->apHash); + sqlite3_free(pCache); +} + +/* +** This function is called during initialization (sqlite3_initialize()) to +** install the default pluggable cache module, assuming the user has not +** already provided an alternative. +*/ +SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ + static const sqlite3_pcache_methods defaultMethods = { + 0, /* pArg */ + pcache1Init, /* xInit */ + pcache1Shutdown, /* xShutdown */ + pcache1Create, /* xCreate */ + pcache1Cachesize, /* xCachesize */ + pcache1Pagecount, /* xPagecount */ + pcache1Fetch, /* xFetch */ + pcache1Unpin, /* xUnpin */ + pcache1Rekey, /* xRekey */ + pcache1Truncate, /* xTruncate */ + pcache1Destroy /* xDestroy */ + }; + sqlite3_config(SQLITE_CONFIG_PCACHE, &defaultMethods); +} + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** This function is called to free superfluous dynamically allocated memory +** held by the pager system. Memory in use by any SQLite pager allocated +** by the current thread may be sqlite3_free()ed. ** -** The ERROR state is entered when an IO or disk-full error (including -** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it -** difficult to be sure that the in-memory pager state (cache contents, -** db size etc.) are consistent with the contents of the file-system. +** nReq is the number of bytes of memory required. Once this much has +** been released, the function returns. The return value is the total number +** of bytes of memory released. +*/ +SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ + int nFree = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + assert( sqlite3_mutex_notheld(pcache1.mutex) ); + if( pcache1.pStart==0 ){ + PgHdr1 *p; + pcache1EnterMutex(&pcache1.grp); + while( (nReq<0 || nFreepLruNext){ + nRecyclable++; + } + *pnCurrent = pcache1.grp.nCurrentPage; + *pnMax = pcache1.grp.nMaxPage; + *pnMin = pcache1.grp.nMinPage; + *pnRecyclable = nRecyclable; +} +#endif + +/************** End of pcache1.c *********************************************/ +/************** Begin file rowset.c ******************************************/ +/* +** 2008 December 3 ** -** Temporary pager files may enter the ERROR state, but in-memory pagers -** cannot. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** For example, if an IO error occurs while performing a rollback, -** the contents of the page-cache may be left in an inconsistent state. -** At this point it would be dangerous to change back to READER state -** (as usually happens after a rollback). Any subsequent readers might -** report database corruption (due to the inconsistent cache), and if -** they upgrade to writers, they may inadvertently corrupt the database -** file. To avoid this hazard, the pager switches into the ERROR state -** instead of READER following such an error. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** Once it has entered the ERROR state, any attempt to use the pager -** to read or write data returns an error. Eventually, once all -** outstanding transactions have been abandoned, the pager is able to -** transition back to OPEN state, discarding the contents of the -** page-cache and any other in-memory state at the same time. Everything -** is reloaded from disk (and, if necessary, hot-journal rollback peformed) -** when a read-transaction is next opened on the pager (transitioning -** the pager into READER state). At that point the system has recovered -** from the error. +************************************************************************* ** -** Specifically, the pager jumps into the ERROR state if: +** This module implements an object we call a "RowSet". ** -** 1. An error occurs while attempting a rollback. This happens in -** function sqlite3PagerRollback(). +** The RowSet object is a collection of rowids. Rowids +** are inserted into the RowSet in an arbitrary order. Inserts +** can be intermixed with tests to see if a given rowid has been +** previously inserted into the RowSet. ** -** 2. An error occurs while attempting to finalize a journal file -** following a commit in function sqlite3PagerCommitPhaseTwo(). +** After all inserts are finished, it is possible to extract the +** elements of the RowSet in sorted order. Once this extraction +** process has started, no new elements may be inserted. ** -** 3. An error occurs while attempting to write to the journal or -** database file in function pagerStress() in order to free up -** memory. +** Hence, the primitive operations for a RowSet are: ** -** In other cases, the error is returned to the b-tree layer. The b-tree -** layer then attempts a rollback operation. If the error condition -** persists, the pager enters the ERROR state via condition (1) above. +** CREATE +** INSERT +** TEST +** SMALLEST +** DESTROY ** -** Condition (3) is necessary because it can be triggered by a read-only -** statement executed within a transaction. In this case, if the error -** code were simply returned to the user, the b-tree layer would not -** automatically attempt a rollback, as it assumes that an error in a -** read-only statement cannot leave the pager in an internally inconsistent -** state. +** The CREATE and DESTROY primitives are the constructor and destructor, +** obviously. The INSERT primitive adds a new element to the RowSet. +** TEST checks to see if an element is already in the RowSet. SMALLEST +** extracts the least value from the RowSet. ** -** * The Pager.errCode variable is set to something other than SQLITE_OK. -** * There are one or more outstanding references to pages (after the -** last reference is dropped the pager should move back to OPEN state). -** * The pager is not an in-memory pager. -** +** The INSERT primitive might allocate additional memory. Memory is +** allocated in chunks so most INSERTs do no allocation. There is an +** upper bound on the size of allocated memory. No memory is freed +** until DESTROY. ** -** Notes: +** The TEST primitive includes a "batch" number. The TEST primitive +** will only see elements that were inserted before the last change +** in the batch number. In other words, if an INSERT occurs between +** two TESTs where the TESTs have the same batch nubmer, then the +** value added by the INSERT will not be visible to the second TEST. +** The initial batch number is zero, so if the very first TEST contains +** a non-zero batch number, it will see all prior INSERTs. ** -** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the -** connection is open in WAL mode. A WAL connection is always in one -** of the first four states. +** No INSERTs may occurs after a SMALLEST. An assertion will fail if +** that is attempted. ** -** * Normally, a connection open in exclusive mode is never in PAGER_OPEN -** state. There are two exceptions: immediately after exclusive-mode has -** been turned on (and before any read or write transactions are -** executed), and when the pager is leaving the "error state". +** The cost of an INSERT is roughly constant. (Sometime new memory +** has to be allocated on an INSERT.) The cost of a TEST with a new +** batch number is O(NlogN) where N is the number of elements in the RowSet. +** The cost of a TEST using the same batch number is O(logN). The cost +** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST +** primitives are constant time. The cost of DESTROY is O(N). ** -** * See also: assert_pager_state(). +** There is an added cost of O(N) when switching between TEST and +** SMALLEST primitives. */ -#define PAGER_OPEN 0 -#define PAGER_READER 1 -#define PAGER_WRITER_LOCKED 2 -#define PAGER_WRITER_CACHEMOD 3 -#define PAGER_WRITER_DBMOD 4 -#define PAGER_WRITER_FINISHED 5 -#define PAGER_ERROR 6 + /* -** The Pager.eLock variable is almost always set to one of the -** following locking-states, according to the lock currently held on -** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. -** This variable is kept up to date as locks are taken and released by -** the pagerLockDb() and pagerUnlockDb() wrappers. -** -** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY -** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not -** the operation was successful. In these circumstances pagerLockDb() and -** pagerUnlockDb() take a conservative approach - eLock is always updated -** when unlocking the file, and only updated when locking the file if the -** VFS call is successful. This way, the Pager.eLock variable may be set -** to a less exclusive (lower) value than the lock that is actually held -** at the system level, but it is never set to a more exclusive value. -** -** This is usually safe. If an xUnlock fails or appears to fail, there may -** be a few redundant xLock() calls or a lock may be held for longer than -** required, but nothing really goes wrong. -** -** The exception is when the database file is unlocked as the pager moves -** from ERROR to OPEN state. At this point there may be a hot-journal file -** in the file-system that needs to be rolled back (as part of a OPEN->SHARED -** transition, by the same pager or any other). If the call to xUnlock() -** fails at this point and the pager is left holding an EXCLUSIVE lock, this -** can confuse the call to xCheckReservedLock() call made later as part -** of hot-journal detection. -** -** xCheckReservedLock() is defined as returning true "if there is a RESERVED -** lock held by this process or any others". So xCheckReservedLock may -** return true because the caller itself is holding an EXCLUSIVE lock (but -** doesn't know it because of a previous error in xUnlock). If this happens -** a hot-journal may be mistaken for a journal being created by an active -** transaction in another process, causing SQLite to read from the database -** without rolling it back. -** -** To work around this, if a call to xUnlock() fails when unlocking the -** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It -** is only changed back to a real locking state after a successful call -** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition -** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK -** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE -** lock on the database file before attempting to roll it back. See function -** PagerSharedLock() for more detail. -** -** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in -** PAGER_OPEN state. -*/ -#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) - -/* -** A macro used for invoking the codec if there is one +** Target size for allocation chunks. */ -#ifdef SQLITE_HAS_CODEC -# define CODEC1(P,D,N,X,E) \ - if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } -# define CODEC2(P,D,N,X,E,O) \ - if( P->xCodec==0 ){ O=(char*)D; }else \ - if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } -#else -# define CODEC1(P,D,N,X,E) /* NO-OP */ -# define CODEC2(P,D,N,X,E,O) O=(char*)D -#endif +#define ROWSET_ALLOCATION_SIZE 1024 /* -** The maximum allowed sector size. 64KiB. If the xSectorsize() method -** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. -** This could conceivably cause corruption following a power failure on -** such a system. This is currently an undocumented limit. +** The number of rowset entries per allocation chunk. */ -#define MAX_SECTOR_SIZE 0x10000 +#define ROWSET_ENTRY_PER_CHUNK \ + ((ROWSET_ALLOCATION_SIZE-8)/sizeof(struct RowSetEntry)) /* -** An instance of the following structure is allocated for each active -** savepoint and statement transaction in the system. All such structures -** are stored in the Pager.aSavepoint[] array, which is allocated and -** resized using sqlite3Realloc(). -** -** When a savepoint is created, the PagerSavepoint.iHdrOffset field is -** set to 0. If a journal-header is written into the main journal while -** the savepoint is active, then iHdrOffset is set to the byte offset -** immediately following the last journal record written into the main -** journal before the journal-header. This is required during savepoint -** rollback (see pagerPlaybackSavepoint()). +** Each entry in a RowSet is an instance of the following object. */ -typedef struct PagerSavepoint PagerSavepoint; -struct PagerSavepoint { - i64 iOffset; /* Starting offset in main journal */ - i64 iHdrOffset; /* See above */ - Bitvec *pInSavepoint; /* Set of pages in this savepoint */ - Pgno nOrig; /* Original number of pages in file */ - Pgno iSubRec; /* Index of first record in sub-journal */ -#ifndef SQLITE_OMIT_WAL - u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ -#endif +struct RowSetEntry { + i64 v; /* ROWID value for this entry */ + struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ + struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ }; /* -** A open page cache is an instance of struct Pager. A description of -** some of the more important member variables follows: -** -** eState -** -** The current 'state' of the pager object. See the comment and state -** diagram above for a description of the pager state. -** -** eLock -** -** For a real on-disk database, the current lock held on the database file - -** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. -** -** For a temporary or in-memory database (neither of which require any -** locks), this variable is always set to EXCLUSIVE_LOCK. Since such -** databases always have Pager.exclusiveMode==1, this tricks the pager -** logic into thinking that it already has all the locks it will ever -** need (and no reason to release them). -** -** In some (obscure) circumstances, this variable may also be set to -** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for -** details. -** -** changeCountDone -** -** This boolean variable is used to make sure that the change-counter -** (the 4-byte header field at byte offset 24 of the database file) is -** not updated more often than necessary. -** -** It is set to true when the change-counter field is updated, which -** can only happen if an exclusive lock is held on the database file. -** It is cleared (set to false) whenever an exclusive lock is -** relinquished on the database file. Each time a transaction is committed, -** The changeCountDone flag is inspected. If it is true, the work of -** updating the change-counter is omitted for the current transaction. -** -** This mechanism means that when running in exclusive mode, a connection -** need only update the change-counter once, for the first transaction -** committed. -** -** setMaster -** -** When PagerCommitPhaseOne() is called to commit a transaction, it may -** (or may not) specify a master-journal name to be written into the -** journal file before it is synced to disk. -** -** Whether or not a journal file contains a master-journal pointer affects -** the way in which the journal file is finalized after the transaction is -** committed or rolled back when running in "journal_mode=PERSIST" mode. -** If a journal file does not contain a master-journal pointer, it is -** finalized by overwriting the first journal header with zeroes. If -** it does contain a master-journal pointer the journal file is finalized -** by truncating it to zero bytes, just as if the connection were -** running in "journal_mode=truncate" mode. -** -** Journal files that contain master journal pointers cannot be finalized -** simply by overwriting the first journal-header with zeroes, as the -** master journal pointer could interfere with hot-journal rollback of any -** subsequently interrupted transaction that reuses the journal file. -** -** The flag is cleared as soon as the journal file is finalized (either -** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the -** journal file from being successfully finalized, the setMaster flag -** is cleared anyway (and the pager will move to ERROR state). -** -** doNotSpill, doNotSyncSpill -** -** These two boolean variables control the behaviour of cache-spills -** (calls made by the pcache module to the pagerStress() routine to -** write cached data to the file-system in order to free up memory). -** -** When doNotSpill is non-zero, writing to the database from pagerStress() -** is disabled altogether. This is done in a very obscure case that -** comes up during savepoint rollback that requires the pcache module -** to allocate a new page to prevent the journal file from being written -** while it is being traversed by code in pager_playback(). -** -** If doNotSyncSpill is non-zero, writing to the database from pagerStress() -** is permitted, but syncing the journal file is not. This flag is set -** by sqlite3PagerWrite() when the file-system sector-size is larger than -** the database page-size in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. -** -** subjInMemory -** -** This is a boolean variable. If true, then any required sub-journal -** is opened as an in-memory journal file. If false, then in-memory -** sub-journals are only used for in-memory pager files. -** -** This variable is updated by the upper layer each time a new -** write-transaction is opened. -** -** dbSize, dbOrigSize, dbFileSize -** -** Variable dbSize is set to the number of pages in the database file. -** It is valid in PAGER_READER and higher states (all states except for -** OPEN and ERROR). -** -** dbSize is set based on the size of the database file, which may be -** larger than the size of the database (the value stored at offset -** 28 of the database header by the btree). If the size of the file -** is not an integer multiple of the page-size, the value stored in -** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). -** Except, any file that is greater than 0 bytes in size is considered -** to have at least one page. (i.e. a 1KB file with 2K page-size leads -** to dbSize==1). -** -** During a write-transaction, if pages with page-numbers greater than -** dbSize are modified in the cache, dbSize is updated accordingly. -** Similarly, if the database is truncated using PagerTruncateImage(), -** dbSize is updated. -** -** Variables dbOrigSize and dbFileSize are valid in states -** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize -** variable at the start of the transaction. It is used during rollback, -** and to determine whether or not pages need to be journalled before -** being modified. -** -** Throughout a write-transaction, dbFileSize contains the size of -** the file on disk in pages. It is set to a copy of dbSize when the -** write-transaction is first opened, and updated when VFS calls are made -** to write or truncate the database file on disk. -** -** The only reason the dbFileSize variable is required is to suppress -** unnecessary calls to xTruncate() after committing a transaction. If, -** when a transaction is committed, the dbFileSize variable indicates -** that the database file is larger than the database image (Pager.dbSize), -** pager_truncate() is called. The pager_truncate() call uses xFilesize() -** to measure the database file on disk, and then truncates it if required. -** dbFileSize is not used when rolling back a transaction. In this case -** pager_truncate() is called unconditionally (which means there may be -** a call to xFilesize() that is not strictly required). In either case, -** pager_truncate() may cause the file to become smaller or larger. -** -** dbHintSize -** -** The dbHintSize variable is used to limit the number of calls made to -** the VFS xFileControl(FCNTL_SIZE_HINT) method. -** -** dbHintSize is set to a copy of the dbSize variable when a -** write-transaction is opened (at the same time as dbFileSize and -** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, -** dbHintSize is increased to the number of pages that correspond to the -** size-hint passed to the method call. See pager_write_pagelist() for -** details. -** -** errCode -** -** The Pager.errCode variable is only ever used in PAGER_ERROR state. It -** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode -** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX -** sub-codes. +** RowSetEntry objects are allocated in large chunks (instances of the +** following structure) to reduce memory allocation overhead. The +** chunks are kept on a linked list so that they can be deallocated +** when the RowSet is destroyed. */ -struct Pager { - sqlite3_vfs *pVfs; /* OS functions to use for IO */ - u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ - u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ - u8 useJournal; /* Use a rollback journal on this file */ - u8 noReadlock; /* Do not bother to obtain readlocks */ - u8 noSync; /* Do not sync the journal if true */ - u8 fullSync; /* Do extra syncs of the journal for robustness */ - u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ - u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ - u8 tempFile; /* zFilename is a temporary file */ - u8 readOnly; /* True for a read-only database */ - u8 memDb; /* True to inhibit all file I/O */ - - /************************************************************************** - ** The following block contains those class members that change during - ** routine opertion. Class members not in this block are either fixed - ** when the pager is first created or else only change when there is a - ** significant mode change (such as changing the page_size, locking_mode, - ** or the journal_mode). From another view, these class members describe - ** the "state" of the pager, while other class members describe the - ** "configuration" of the pager. - */ - u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ - u8 eLock; /* Current lock held on database file */ - u8 changeCountDone; /* Set after incrementing the change-counter */ - u8 setMaster; /* True if a m-j name has been written to jrnl */ - u8 doNotSpill; /* Do not spill the cache when non-zero */ - u8 doNotSyncSpill; /* Do not do a spill that requires jrnl sync */ - u8 subjInMemory; /* True to use in-memory sub-journals */ - Pgno dbSize; /* Number of pages in the database */ - Pgno dbOrigSize; /* dbSize before the current transaction */ - Pgno dbFileSize; /* Number of pages in the database file */ - Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ - int errCode; /* One of several kinds of errors */ - int nRec; /* Pages journalled since last j-header written */ - u32 cksumInit; /* Quasi-random value added to every checksum */ - u32 nSubRec; /* Number of records written to sub-journal */ - Bitvec *pInJournal; /* One bit for each page in the database file */ - sqlite3_file *fd; /* File descriptor for database */ - sqlite3_file *jfd; /* File descriptor for main journal */ - sqlite3_file *sjfd; /* File descriptor for sub-journal */ - i64 journalOff; /* Current write offset in the journal file */ - i64 journalHdr; /* Byte offset to previous journal header */ - sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ - PagerSavepoint *aSavepoint; /* Array of active savepoints */ - int nSavepoint; /* Number of elements in aSavepoint[] */ - char dbFileVers[16]; /* Changes whenever database file changes */ - /* - ** End of the routinely-changing class members - ***************************************************************************/ - - u16 nExtra; /* Add this many bytes to each in-memory page */ - i16 nReserve; /* Number of unused bytes at end of each page */ - u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ - u32 sectorSize; /* Assumed sector size during rollback */ - int pageSize; /* Number of bytes in a page */ - Pgno mxPgno; /* Maximum allowed size of the database */ - i64 journalSizeLimit; /* Size limit for persistent journal files */ - char *zFilename; /* Name of the database file */ - char *zJournal; /* Name of the journal file */ - int (*xBusyHandler)(void*); /* Function to call when busy */ - void *pBusyHandlerArg; /* Context argument for xBusyHandler */ -#ifdef SQLITE_TEST - int nHit, nMiss; /* Cache hits and missing */ - int nRead, nWrite; /* Database pages read/written */ -#endif - void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ -#ifdef SQLITE_HAS_CODEC - void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ - void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ - void (*xCodecFree)(void*); /* Destructor for the codec */ - void *pCodec; /* First argument to xCodec... methods */ -#endif - char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ - PCache *pPCache; /* Pointer to page cache object */ -#ifndef SQLITE_OMIT_WAL - Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ - char *zWal; /* File name for write-ahead log */ -#endif +struct RowSetChunk { + struct RowSetChunk *pNextChunk; /* Next chunk on list of them all */ + struct RowSetEntry aEntry[ROWSET_ENTRY_PER_CHUNK]; /* Allocated entries */ }; /* -** The following global variables hold counters used for -** testing purposes only. These variables do not exist in -** a non-testing build. These variables are not thread-safe. -*/ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ -SQLITE_API int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ -SQLITE_API int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ -# define PAGER_INCR(v) v++ -#else -# define PAGER_INCR(v) -#endif - - - -/* -** Journal files begin with the following magic string. The data -** was obtained from /dev/random. It is used only as a sanity check. -** -** Since version 2.8.0, the journal format contains additional sanity -** checking information. If the power fails while the journal is being -** written, semi-random garbage data might appear in the journal -** file after power is restored. If an attempt is then made -** to roll the journal back, the database could be corrupted. The additional -** sanity checking data is an attempt to discover the garbage in the -** journal and ignore it. +** A RowSet in an instance of the following structure. ** -** The sanity checking information for the new journal format consists -** of a 32-bit checksum on each page of data. The checksum covers both -** the page number and the pPager->pageSize bytes of data for the page. -** This cksum is initialized to a 32-bit random value that appears in the -** journal file right after the header. The random initializer is important, -** because garbage data that appears at the end of a journal is likely -** data that was once in other files that have now been deleted. If the -** garbage data came from an obsolete journal file, the checksums might -** be correct. But by initializing the checksum to random value which -** is different for every journal, we minimize that risk. +** A typedef of this structure if found in sqliteInt.h. */ -static const unsigned char aJournalMagic[] = { - 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7, +struct RowSet { + struct RowSetChunk *pChunk; /* List of all chunk allocations */ + sqlite3 *db; /* The database connection */ + struct RowSetEntry *pEntry; /* List of entries using pRight */ + struct RowSetEntry *pLast; /* Last entry on the pEntry list */ + struct RowSetEntry *pFresh; /* Source of new entry objects */ + struct RowSetEntry *pTree; /* Binary tree of entries */ + u16 nFresh; /* Number of objects on pFresh */ + u8 isSorted; /* True if pEntry is sorted */ + u8 iBatch; /* Current insert batch */ }; /* -** The size of the of each page record in the journal is given by -** the following macro. -*/ -#define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) - -/* -** The journal header size for this pager. This is usually the same -** size as a single disk sector. See also setSectorSize(). +** Turn bulk memory into a RowSet object. N bytes of memory +** are available at pSpace. The db pointer is used as a memory context +** for any subsequent allocations that need to occur. +** Return a pointer to the new RowSet object. +** +** It must be the case that N is sufficient to make a Rowset. If not +** an assertion fault occurs. +** +** If N is larger than the minimum, use the surplus as an initial +** allocation of entries available to be filled. */ -#define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ + RowSet *p; + assert( N >= ROUND8(sizeof(*p)) ); + p = pSpace; + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pTree = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->isSorted = 1; + p->iBatch = 0; + return p; +} /* -** The macro MEMDB is true if we are dealing with an in-memory database. -** We do this as a macro so that if the SQLITE_OMIT_MEMORYDB macro is set, -** the value of MEMDB will be a constant and the compiler will optimize -** out code that would never execute. +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. */ -#ifdef SQLITE_OMIT_MEMORYDB -# define MEMDB 0 -#else -# define MEMDB pPager->memDb -#endif +SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ + struct RowSetChunk *pChunk, *pNextChunk; + for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ + pNextChunk = pChunk->pNextChunk; + sqlite3DbFree(p->db, pChunk); + } + p->pChunk = 0; + p->nFresh = 0; + p->pEntry = 0; + p->pLast = 0; + p->pTree = 0; + p->isSorted = 1; +} /* -** The maximum legal page number is (2^31 - 1). +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. */ -#define PAGER_MAX_PGNO 2147483647 +SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; /* The new entry */ + struct RowSetEntry *pLast; /* The last prior entry */ + assert( p!=0 ); + if( p->nFresh==0 ){ + struct RowSetChunk *pNew; + pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); + if( pNew==0 ){ + return; + } + pNew->pNextChunk = p->pChunk; + p->pChunk = pNew; + p->pFresh = pNew->aEntry; + p->nFresh = ROWSET_ENTRY_PER_CHUNK; + } + pEntry = p->pFresh++; + p->nFresh--; + pEntry->v = rowid; + pEntry->pRight = 0; + pLast = p->pLast; + if( pLast ){ + if( p->isSorted && rowid<=pLast->v ){ + p->isSorted = 0; + } + pLast->pRight = pEntry; + }else{ + assert( p->pEntry==0 ); /* Fires if INSERT after SMALLEST */ + p->pEntry = pEntry; + } + p->pLast = pEntry; +} /* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. -** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of +** Merge two lists of RowSetEntry objects. Remove duplicates. ** -** if( pPager->jfd->pMethods ){ ... +** The input lists are connected via pRight pointers and are +** assumed to each already be in sorted order. */ -#define isOpen(pFd) ((pFd)->pMethods) +static struct RowSetEntry *rowSetMerge( + struct RowSetEntry *pA, /* First sorted list to be merged */ + struct RowSetEntry *pB /* Second sorted list to be merged */ +){ + struct RowSetEntry head; + struct RowSetEntry *pTail; -/* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. -*/ -#ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); + pTail = &head; + while( pA && pB ){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + assert( pB->pRight==0 || pB->v<=pB->pRight->v ); + if( pA->vv ){ + pTail->pRight = pA; + pA = pA->pRight; + pTail = pTail->pRight; + }else if( pB->vv ){ + pTail->pRight = pB; + pB = pB->pRight; + pTail = pTail->pRight; + }else{ + pA = pA->pRight; + } + } + if( pA ){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + pTail->pRight = pA; + }else{ + assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); + pTail->pRight = pB; + } + return head.pRight; } -#else -# define pagerUseWal(x) 0 -# define pagerRollbackWal(x) 0 -# define pagerWalFrames(v,w,x,y,z) 0 -# define pagerOpenWalIfPresent(z) SQLITE_OK -# define pagerBeginReadTransaction(z) SQLITE_OK -#endif -#ifndef NDEBUG /* -** Usage: -** -** assert( assert_pager_state(pPager) ); -** -** This function runs many asserts to try to find inconsistencies in -** the internal state of the Pager object. -*/ -static int assert_pager_state(Pager *p){ - Pager *pPager = p; - - /* State must be valid. */ - assert( p->eState==PAGER_OPEN - || p->eState==PAGER_READER - || p->eState==PAGER_WRITER_LOCKED - || p->eState==PAGER_WRITER_CACHEMOD - || p->eState==PAGER_WRITER_DBMOD - || p->eState==PAGER_WRITER_FINISHED - || p->eState==PAGER_ERROR - ); - - /* Regardless of the current state, a temp-file connection always behaves - ** as if it has an exclusive lock on the database file. It never updates - ** the change-counter field, so the changeCountDone flag is always set. - */ - assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); - assert( p->tempFile==0 || pPager->changeCountDone ); - - /* If the useJournal flag is clear, the journal-mode must be "OFF". - ** And if the journal-mode is "OFF", the journal file must not be open. - */ - assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); - assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); +** Sort all elements on the pEntry list of the RowSet into ascending order. +*/ +static void rowSetSort(RowSet *p){ + unsigned int i; + struct RowSetEntry *pEntry; + struct RowSetEntry *aBucket[40]; - /* Check that MEMDB implies noSync. And an in-memory journal. Since - ** this means an in-memory pager performs no IO at all, it cannot encounter - ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing - ** a journal file. (although the in-memory journal implementation may - ** return SQLITE_IOERR_NOMEM while the journal file is being written). It - ** is therefore not possible for an in-memory pager to enter the ERROR - ** state. - */ - if( MEMDB ){ - assert( p->noSync ); - assert( p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_MEMORY - ); - assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); - assert( pagerUseWal(p)==0 ); + assert( p->isSorted==0 ); + memset(aBucket, 0, sizeof(aBucket)); + while( p->pEntry ){ + pEntry = p->pEntry; + p->pEntry = pEntry->pRight; + pEntry->pRight = 0; + for(i=0; aBucket[i]; i++){ + pEntry = rowSetMerge(aBucket[i], pEntry); + aBucket[i] = 0; + } + aBucket[i] = pEntry; } - - /* If changeCountDone is set, a RESERVED lock or greater must be held - ** on the file. - */ - assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); - assert( p->eLock!=PENDING_LOCK ); - - switch( p->eState ){ - case PAGER_OPEN: - assert( !MEMDB ); - assert( pPager->errCode==SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); - break; - - case PAGER_READER: - assert( pPager->errCode==SQLITE_OK ); - assert( p->eLock!=UNKNOWN_LOCK ); - assert( p->eLock>=SHARED_LOCK || p->noReadlock ); - break; - - case PAGER_WRITER_LOCKED: - assert( p->eLock!=UNKNOWN_LOCK ); - assert( pPager->errCode==SQLITE_OK ); - if( !pagerUseWal(pPager) ){ - assert( p->eLock>=RESERVED_LOCK ); - } - assert( pPager->dbSize==pPager->dbOrigSize ); - assert( pPager->dbOrigSize==pPager->dbFileSize ); - assert( pPager->dbOrigSize==pPager->dbHintSize ); - assert( pPager->setMaster==0 ); - break; - - case PAGER_WRITER_CACHEMOD: - assert( p->eLock!=UNKNOWN_LOCK ); - assert( pPager->errCode==SQLITE_OK ); - if( !pagerUseWal(pPager) ){ - /* It is possible that if journal_mode=wal here that neither the - ** journal file nor the WAL file are open. This happens during - ** a rollback transaction that switches from journal_mode=off - ** to journal_mode=wal. - */ - assert( p->eLock>=RESERVED_LOCK ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL - ); - } - assert( pPager->dbOrigSize==pPager->dbFileSize ); - assert( pPager->dbOrigSize==pPager->dbHintSize ); - break; - - case PAGER_WRITER_DBMOD: - assert( p->eLock==EXCLUSIVE_LOCK ); - assert( pPager->errCode==SQLITE_OK ); - assert( !pagerUseWal(pPager) ); - assert( p->eLock>=EXCLUSIVE_LOCK ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL - ); - assert( pPager->dbOrigSize<=pPager->dbHintSize ); - break; - - case PAGER_WRITER_FINISHED: - assert( p->eLock==EXCLUSIVE_LOCK ); - assert( pPager->errCode==SQLITE_OK ); - assert( !pagerUseWal(pPager) ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL - ); - break; - - case PAGER_ERROR: - /* There must be at least one outstanding reference to the pager if - ** in ERROR state. Otherwise the pager should have already dropped - ** back to OPEN state. - */ - assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); - break; + pEntry = 0; + for(i=0; ipEntry = pEntry; + p->pLast = 0; + p->isSorted = 1; } -#endif /* ifndef NDEBUG */ - -#ifdef SQLITE_DEBUG -/* -** Return a pointer to a human readable string in a static buffer -** containing the state of the Pager object passed as an argument. This -** is intended to be used within debuggers. For example, as an alternative -** to "print *pPager" in gdb: -** -** (gdb) printf "%s", print_pager_state(pPager) -*/ -static char *print_pager_state(Pager *p){ - static char zRet[1024]; - - sqlite3_snprintf(1024, zRet, - "Filename: %s\n" - "State: %s errCode=%d\n" - "Lock: %s\n" - "Locking mode: locking_mode=%s\n" - "Journal mode: journal_mode=%s\n" - "Backing store: tempFile=%d memDb=%d useJournal=%d\n" - "Journal: journalOff=%lld journalHdr=%lld\n" - "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" - , p->zFilename - , p->eState==PAGER_OPEN ? "OPEN" : - p->eState==PAGER_READER ? "READER" : - p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : - p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : - p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : - p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : - p->eState==PAGER_ERROR ? "ERROR" : "?error?" - , (int)p->errCode - , p->eLock==NO_LOCK ? "NO_LOCK" : - p->eLock==RESERVED_LOCK ? "RESERVED" : - p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : - p->eLock==SHARED_LOCK ? "SHARED" : - p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" - , p->exclusiveMode ? "exclusive" : "normal" - , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : - p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : - p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : - p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : - p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : - p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" - , (int)p->tempFile, (int)p->memDb, (int)p->useJournal - , p->journalOff, p->journalHdr - , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize - ); - return zRet; -} -#endif /* -** Return true if it is necessary to write page *pPg into the sub-journal. -** A page needs to be written into the sub-journal if there exists one -** or more open savepoints for which: -** -** * The page-number is less than or equal to PagerSavepoint.nOrig, and -** * The bit corresponding to the page-number is not set in -** PagerSavepoint.pInSavepoint. +** The input, pIn, is a binary tree (or subtree) of RowSetEntry objects. +** Convert this tree into a linked list connected by the pRight pointers +** and return pointers to the first and last elements of the new list. */ -static int subjRequiresPage(PgHdr *pPg){ - Pgno pgno = pPg->pgno; - Pager *pPager = pPg->pPager; - int i; - for(i=0; inSavepoint; i++){ - PagerSavepoint *p = &pPager->aSavepoint[i]; - if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ - return 1; - } +static void rowSetTreeToList( + struct RowSetEntry *pIn, /* Root of the input tree */ + struct RowSetEntry **ppFirst, /* Write head of the output list here */ + struct RowSetEntry **ppLast /* Write tail of the output list here */ +){ + assert( pIn!=0 ); + if( pIn->pLeft ){ + struct RowSetEntry *p; + rowSetTreeToList(pIn->pLeft, ppFirst, &p); + p->pRight = pIn; + }else{ + *ppFirst = pIn; } - return 0; + if( pIn->pRight ){ + rowSetTreeToList(pIn->pRight, &pIn->pRight, ppLast); + }else{ + *ppLast = pIn; + } + assert( (*ppLast)->pRight==0 ); } -/* -** Return true if the page is already in the journal file. -*/ -static int pageInJournal(PgHdr *pPg){ - return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno); -} /* -** Read a 32-bit integer from the given file descriptor. Store the integer -** that is read in *pRes. Return SQLITE_OK if everything worked, or an -** error code is something goes wrong. +** Convert a sorted list of elements (connected by pRight) into a binary +** tree with depth of iDepth. A depth of 1 means the tree contains a single +** node taken from the head of *ppList. A depth of 2 means a tree with +** three nodes. And so forth. ** -** All values are stored on disk as big-endian. +** Use as many entries from the input list as required and update the +** *ppList to point to the unused elements of the list. If the input +** list contains too few elements, then construct an incomplete tree +** and leave *ppList set to NULL. +** +** Return a pointer to the root of the constructed binary tree. */ -static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ - unsigned char ac[4]; - int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset); - if( rc==SQLITE_OK ){ - *pRes = sqlite3Get4byte(ac); +static struct RowSetEntry *rowSetNDeepTree( + struct RowSetEntry **ppList, + int iDepth +){ + struct RowSetEntry *p; /* Root of the new tree */ + struct RowSetEntry *pLeft; /* Left subtree */ + if( *ppList==0 ){ + return 0; } - return rc; + if( iDepth==1 ){ + p = *ppList; + *ppList = p->pRight; + p->pLeft = p->pRight = 0; + return p; + } + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + return p; } /* -** Write a 32-bit integer into a string buffer in big-endian byte order. +** Convert a sorted list of elements into a binary tree. Make the tree +** as deep as it needs to be in order to contain the entire list. */ -#define put32bits(A,B) sqlite3Put4byte((u8*)A,B) +static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ + int iDepth; /* Depth of the tree so far */ + struct RowSetEntry *p; /* Current tree root */ + struct RowSetEntry *pLeft; /* Left subtree */ + assert( pList!=0 ); + p = pList; + pList = p->pRight; + p->pLeft = p->pRight = 0; + for(iDepth=1; pList; iDepth++){ + pLeft = p; + p = pList; + pList = p->pRight; + p->pLeft = pLeft; + p->pRight = rowSetNDeepTree(&pList, iDepth); + } + return p; +} /* -** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK -** on success or an error code is something goes wrong. +** Convert the list in p->pEntry into a sorted list if it is not +** sorted already. If there is a binary tree on p->pTree, then +** convert it into a list too and merge it into the p->pEntry list. */ -static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ - char ac[4]; - put32bits(ac, val); - return sqlite3OsWrite(fd, ac, 4, offset); +static void rowSetToList(RowSet *p){ + if( !p->isSorted ){ + rowSetSort(p); + } + if( p->pTree ){ + struct RowSetEntry *pHead, *pTail; + rowSetTreeToList(p->pTree, &pHead, &pTail); + p->pTree = 0; + p->pEntry = rowSetMerge(p->pEntry, pHead); + } } /* -** Unlock the database file to level eLock, which must be either NO_LOCK -** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() -** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** Extract the smallest element from the RowSet. +** Write the element into *pRowid. Return 1 on success. Return +** 0 if the RowSet is already empty. ** -** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is -** called, do not modify it. See the comment above the #define of -** UNKNOWN_LOCK for an explanation of this. +** After this routine has been called, the sqlite3RowSetInsert() +** routine may not be called again. */ -static int pagerUnlockDb(Pager *pPager, int eLock){ - int rc = SQLITE_OK; - - assert( !pPager->exclusiveMode || pPager->eLock==eLock ); - assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); - assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); - if( isOpen(pPager->fd) ){ - assert( pPager->eLock>=eLock ); - rc = sqlite3OsUnlock(pPager->fd, eLock); - if( pPager->eLock!=UNKNOWN_LOCK ){ - pPager->eLock = (u8)eLock; +SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ + rowSetToList(p); + if( p->pEntry ){ + *pRowid = p->pEntry->v; + p->pEntry = p->pEntry->pRight; + if( p->pEntry==0 ){ + sqlite3RowSetClear(p); } - IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + return 1; + }else{ + return 0; } - return rc; } /* -** Lock the database file to level eLock, which must be either SHARED_LOCK, -** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the -** Pager.eLock variable to the new locking state. -** -** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is -** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. -** See the comment above the #define of UNKNOWN_LOCK for an explanation -** of this. +** Check to see if element iRowid was inserted into the the rowset as +** part of any insert batch prior to iBatch. Return 1 or 0. */ -static int pagerLockDb(Pager *pPager, int eLock){ - int rc = SQLITE_OK; - - assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); - if( pPager->eLockeLock==UNKNOWN_LOCK ){ - rc = sqlite3OsLock(pPager->fd, eLock); - if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ - pPager->eLock = (u8)eLock; - IOTRACE(("LOCK %p %d\n", pPager, eLock)) +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 iRowid){ + struct RowSetEntry *p; + if( iBatch!=pRowSet->iBatch ){ + if( pRowSet->pEntry ){ + rowSetToList(pRowSet); + pRowSet->pTree = rowSetListToTree(pRowSet->pEntry); + pRowSet->pEntry = 0; + pRowSet->pLast = 0; } + pRowSet->iBatch = iBatch; } - return rc; + p = pRowSet->pTree; + while( p ){ + if( p->vpRight; + }else if( p->v>iRowid ){ + p = p->pLeft; + }else{ + return 1; + } + } + return 0; } +/************** End of rowset.c **********************************************/ +/************** Begin file pager.c *******************************************/ /* -** This function determines whether or not the atomic-write optimization -** can be used with this pager. The optimization can be used if: +** 2001 September 15 ** -** (a) the value returned by OsDeviceCharacteristics() indicates that -** a database page may be written atomically, and -** (b) the value returned by OsSectorSize() is less than or equal -** to the page size. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** The optimization is also always enabled for temporary files. It is -** an error to call this function if pPager is opened on an in-memory -** database. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** If the optimization cannot be used, 0 is returned. If it can be used, -** then the value returned is the size of the journal file when it -** contains rollback data for exactly one page. +************************************************************************* +** This is the implementation of the page cache subsystem or "pager". +** +** The pager is used to access a database disk file. It implements +** atomic commit and rollback through the use of a journal file that +** is separate from the database file. The pager also implements file +** locking to prevent two processes from writing the same database +** file simultaneously, or one process from reading the database while +** another is writing. +*/ +#ifndef SQLITE_OMIT_DISKIO +/************** Include wal.h in the middle of pager.c ***********************/ +/************** Begin file wal.h *********************************************/ +/* +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface to the write-ahead logging +** system. Refer to the comments below and the header comment attached to +** the implementation of each function in log.c for further details. */ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE -static int jrnlBufferSize(Pager *pPager){ - assert( !MEMDB ); - if( !pPager->tempFile ){ - int dc; /* Device characteristics */ - int nSector; /* Sector size */ - int szPage; /* Page size */ - assert( isOpen(pPager->fd) ); - dc = sqlite3OsDeviceCharacteristics(pPager->fd); - nSector = pPager->sectorSize; - szPage = pPager->pageSize; +#ifndef _WAL_H_ +#define _WAL_H_ - assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); - assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); - if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){ - return 0; - } - } - return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); -} -#endif +#ifdef SQLITE_OMIT_WAL +# define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalLimit(x,y) +# define sqlite3WalClose(w,x,y,z) 0 +# define sqlite3WalBeginReadTransaction(y,z) 0 +# define sqlite3WalEndReadTransaction(z) +# define sqlite3WalRead(v,w,x,y,z) 0 +# define sqlite3WalDbsize(y) 0 +# define sqlite3WalBeginWriteTransaction(y) 0 +# define sqlite3WalEndWriteTransaction(x) 0 +# define sqlite3WalUndo(x,y,z) 0 +# define sqlite3WalSavepoint(y,z) +# define sqlite3WalSavepointUndo(y,z) 0 +# define sqlite3WalFrames(u,v,w,x,y,z) 0 +# define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 +# define sqlite3WalCallback(z) 0 +# define sqlite3WalExclusiveMode(y,z) 0 +# define sqlite3WalHeapMemory(z) 0 +#else -/* -** If SQLITE_CHECK_PAGES is defined then we do some sanity checking -** on the cache using a hash function. This is used for testing -** and debugging only. -*/ -#ifdef SQLITE_CHECK_PAGES -/* -** Return a 32-bit hash of the page data for pPage. -*/ -static u32 pager_datahash(int nByte, unsigned char *pData){ - u32 hash = 0; - int i; - for(i=0; ipPager->pageSize, (unsigned char *)pPage->pData); -} -static void pager_set_pagehash(PgHdr *pPage){ - pPage->pageHash = pager_pagehash(pPage); -} +#define WAL_SAVEPOINT_NDATA 4 -/* -** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES -** is defined, and NDEBUG is not defined, an assert() statement checks -** that the page is either dirty or still matches the calculated page-hash. +/* Connection to a write-ahead log (WAL) file. +** There is one object of this type for each pager. */ -#define CHECK_PAGE(x) checkPage(x) -static void checkPage(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - assert( pPager->eState!=PAGER_ERROR ); - assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); -} +typedef struct Wal Wal; -#else -#define pager_datahash(X,Y) 0 -#define pager_pagehash(X) 0 -#define pager_set_pagehash(X) -#define CHECK_PAGE(x) -#endif /* SQLITE_CHECK_PAGES */ - -/* -** When this is called the journal file for pager pPager must be open. -** This function attempts to read a master journal file name from the -** end of the file and, if successful, copies it into memory supplied -** by the caller. See comments above writeMasterJournal() for the format -** used to store a master journal file name at the end of a journal file. -** -** zMaster must point to a buffer of at least nMaster bytes allocated by -** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is -** enough space to write the master journal name). If the master journal -** name in the journal is longer than nMaster bytes (including a -** nul-terminator), then this is handled as if no master journal name -** were present in the journal. -** -** If a master journal file name is present at the end of the journal -** file, then it is copied into the buffer pointed to by zMaster. A -** nul-terminator byte is appended to the buffer following the master -** journal file name. -** -** If it is determined that no master journal file name is present -** zMaster[0] is set to 0 and SQLITE_OK returned. -** -** If an error occurs while reading from the journal file, an SQLite -** error code is returned. -*/ -static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ - int rc; /* Return code */ - u32 len; /* Length in bytes of master journal name */ - i64 szJ; /* Total size in bytes of journal file pJrnl */ - u32 cksum; /* MJ checksum value read from journal */ - u32 u; /* Unsigned loop counter */ - unsigned char aMagic[8]; /* A buffer to hold the magic header */ - zMaster[0] = '\0'; - - if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) - || szJ<16 - || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) - || len>=nMaster - || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) - || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) - || memcmp(aMagic, aJournalMagic, 8) - || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) - ){ - return rc; - } +/* Open and close a connection to a write-ahead log. */ +SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); - /* See if the checksum matches the master journal name */ - for(u=0; ujournalOff, assuming a sector -** size of pPager->sectorSize bytes. -** -** i.e for a sector size of 512: -** -** Pager.journalOff Return value -** --------------------------------------- -** 0 0 -** 512 512 -** 100 512 -** 2000 2048 -** +/* Used by readers to open (lock) and close (unlock) a snapshot. A +** snapshot is like a read-transaction. It is the state of the database +** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and +** preserves the current state even if the other threads or processes +** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the +** transaction and releases the lock. */ -static i64 journalHdrOffset(Pager *pPager){ - i64 offset = 0; - i64 c = pPager->journalOff; - if( c ){ - offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager); - } - assert( offset%JOURNAL_HDR_SZ(pPager)==0 ); - assert( offset>=c ); - assert( (offset-c)jfd) ); - if( pPager->journalOff ){ - const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ +/* Read a page from the write-ahead log, if it is present. */ +SQLITE_PRIVATE int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); - IOTRACE(("JZEROHDR %p\n", pPager)) - if( doTruncate || iLimit==0 ){ - rc = sqlite3OsTruncate(pPager->jfd, 0); - }else{ - static const char zeroHdr[28] = {0}; - rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); - } - if( rc==SQLITE_OK && !pPager->noSync ){ - rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); - } +/* If the WAL is not empty, return the size of the database. */ +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal); - /* At this point the transaction is committed but the write lock - ** is still held on the file. If there is a size limit configured for - ** the persistent journal and the journal file currently consumes more - ** space than that limit allows for, truncate it now. There is no need - ** to sync the file following this operation. - */ - if( rc==SQLITE_OK && iLimit>0 ){ - i64 sz; - rc = sqlite3OsFileSize(pPager->jfd, &sz); - if( rc==SQLITE_OK && sz>iLimit ){ - rc = sqlite3OsTruncate(pPager->jfd, iLimit); - } - } - } - return rc; -} +/* Obtain or release the WRITER lock. */ +SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal); -/* -** The journal file must be open when this routine is called. A journal -** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the -** current location. -** -** The format for the journal header is as follows: -** - 8 bytes: Magic identifying journal format. -** - 4 bytes: Number of records in journal, or -1 no-sync mode is on. -** - 4 bytes: Random number used for page hash. -** - 4 bytes: Initial database page count. -** - 4 bytes: Sector size used by the process that wrote this journal. -** - 4 bytes: Database page size. -** -** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. -*/ -static int writeJournalHdr(Pager *pPager){ - int rc = SQLITE_OK; /* Return code */ - char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ - u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ - u32 nWrite; /* Bytes of header sector written */ - int ii; /* Loop counter */ +/* Undo any frames written (but not committed) to the log */ +SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); - assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ +/* Return an integer that records the current (uncommitted) write +** position in the WAL */ +SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); - if( nHeader>JOURNAL_HDR_SZ(pPager) ){ - nHeader = JOURNAL_HDR_SZ(pPager); - } +/* Move the write position of the WAL back to iFrame. Called in +** response to a ROLLBACK TO command. */ +SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); - /* If there are active savepoints and any of them were created - ** since the most recent journal header was written, update the - ** PagerSavepoint.iHdrOffset fields now. - */ - for(ii=0; iinSavepoint; ii++){ - if( pPager->aSavepoint[ii].iHdrOffset==0 ){ - pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff; - } - } +/* Write a frame or frames to the log. */ +SQLITE_PRIVATE int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); - pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); +/* Copy pages from the log to the database file */ +SQLITE_PRIVATE int sqlite3WalCheckpoint( + Wal *pWal, /* Write-ahead log connection */ + int eMode, /* One of PASSIVE, FULL and RESTART */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of buffer nBuf */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ +); - /* - ** Write the nRec Field - the number of page records that follow this - ** journal header. Normally, zero is written to this value at this time. - ** After the records are added to the journal (and the journal synced, - ** if in full-sync mode), the zero is overwritten with the true number - ** of records (see syncJournal()). - ** - ** A faster alternative is to write 0xFFFFFFFF to the nRec field. When - ** reading the journal this value tells SQLite to assume that the - ** rest of the journal file contains valid page records. This assumption - ** is dangerous, as if a failure occurred whilst writing to the journal - ** file it may contain some garbage data. There are two scenarios - ** where this risk can be ignored: - ** - ** * When the pager is in no-sync mode. Corruption can follow a - ** power failure in this case anyway. - ** - ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees - ** that garbage data is never appended to the journal file. - */ - assert( isOpen(pPager->fd) || pPager->noSync ); - if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) - || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) - ){ - memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); - put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); - }else{ - memset(zHeader, 0, sizeof(aJournalMagic)+4); - } +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ +SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal); - /* The random check-hash initialiser */ - sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); - put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); - /* The initial database size */ - put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); - /* The assumed sector size for this process */ - put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); +/* Tell the wal layer that an EXCLUSIVE lock has been obtained (or released) +** by the pager layer on the database file. +*/ +SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); - /* The page size */ - put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize); +/* Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); - /* Initializing the tail of the buffer is not necessary. Everything - ** works find if the following memset() is omitted. But initializing - ** the memory prevents valgrind from complaining, so we are willing to - ** take the performance hit. - */ - memset(&zHeader[sizeof(aJournalMagic)+20], 0, - nHeader-(sizeof(aJournalMagic)+20)); +#endif /* ifndef SQLITE_OMIT_WAL */ +#endif /* _WAL_H_ */ - /* In theory, it is only necessary to write the 28 bytes that the - ** journal header consumes to the journal file here. Then increment the - ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next - ** record is written to the following sector (leaving a gap in the file - ** that will be implicitly filled in by the OS). - ** - ** However it has been discovered that on some systems this pattern can - ** be significantly slower than contiguously writing data to the file, - ** even if that means explicitly writing data to the block of - ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what - ** is done. - ** - ** The loop is required here in case the sector-size is larger than the - ** database page size. Since the zHeader buffer is only Pager.pageSize - ** bytes in size, more than one call to sqlite3OsWrite() may be required - ** to populate the entire journal header sector. - */ - for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) - rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); - assert( pPager->journalHdr <= pPager->journalOff ); - pPager->journalOff += nHeader; - } +/************** End of wal.h *************************************************/ +/************** Continuing where we left off in pager.c **********************/ - return rc; -} -/* -** The journal file must be open when this is called. A journal header file -** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal -** file. The current location in the journal file is given by -** pPager->journalOff. See comments above function writeJournalHdr() for -** a description of the journal header format. +/******************* NOTES ON THE DESIGN OF THE PAGER ************************ ** -** If the header is read successfully, *pNRec is set to the number of -** page records following this header and *pDbSize is set to the size of the -** database before the transaction began, in pages. Also, pPager->cksumInit -** is set to the value read from the journal header. SQLITE_OK is returned -** in this case. +** This comment block describes invariants that hold when using a rollback +** journal. These invariants do not apply for journal_mode=WAL, +** journal_mode=MEMORY, or journal_mode=OFF. ** -** If the journal header file appears to be corrupted, SQLITE_DONE is -** returned and *pNRec and *PDbSize are undefined. If JOURNAL_HDR_SZ bytes -** cannot be read from the journal file an error code is returned. -*/ -static int readJournalHdr( - Pager *pPager, /* Pager object */ - int isHot, - i64 journalSize, /* Size of the open journal file in bytes */ - u32 *pNRec, /* OUT: Value read from the nRec field */ - u32 *pDbSize /* OUT: Value of original database size field */ -){ - int rc; /* Return code */ - unsigned char aMagic[8]; /* A buffer to hold the magic header */ - i64 iHdrOff; /* Offset of journal header being read */ - - assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ - - /* Advance Pager.journalOff to the start of the next sector. If the - ** journal file is too small for there to be a header stored at this - ** point, return SQLITE_DONE. - */ - pPager->journalOff = journalHdrOffset(pPager); - if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){ - return SQLITE_DONE; - } - iHdrOff = pPager->journalOff; - - /* Read in the first 8 bytes of the journal header. If they do not match - ** the magic string found at the start of each journal header, return - ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise, - ** proceed. - */ - if( isHot || iHdrOff!=pPager->journalHdr ){ - rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff); - if( rc ){ - return rc; - } - if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){ - return SQLITE_DONE; - } - } - - /* Read the first three 32-bit fields of the journal header: The nRec - ** field, the checksum-initializer and the database size at the start - ** of the transaction. Return an error code if anything goes wrong. - */ - if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec)) - || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+12, &pPager->cksumInit)) - || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+16, pDbSize)) - ){ - return rc; - } - - if( pPager->journalOff==0 ){ - u32 iPageSize; /* Page-size field of journal header */ - u32 iSectorSize; /* Sector-size field of journal header */ - - /* Read the page-size and sector-size journal header fields. */ - if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) - || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize)) - ){ - return rc; - } - - /* Versions of SQLite prior to 3.5.8 set the page-size field of the - ** journal header to zero. In this case, assume that the Pager.pageSize - ** variable is already set to the correct page size. - */ - if( iPageSize==0 ){ - iPageSize = pPager->pageSize; - } - - /* Check that the values read from the page-size and sector-size fields - ** are within range. To be 'in range', both values need to be a power - ** of two greater than or equal to 512 or 32, and not greater than their - ** respective compile time maximum limits. - */ - if( iPageSize<512 || iSectorSize<32 - || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE - || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 - ){ - /* If the either the page-size or sector-size in the journal-header is - ** invalid, then the process that wrote the journal-header must have - ** crashed before the header was synced. In this case stop reading - ** the journal file here. - */ - return SQLITE_DONE; - } - - /* Update the page-size to match the value read from the journal. - ** Use a testcase() macro to make sure that malloc failure within - ** PagerSetPagesize() is tested. - */ - rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); - testcase( rc!=SQLITE_OK ); - - /* Update the assumed sector-size to match the value used by - ** the process that created this journal. If this journal was - ** created by a process other than this one, then this routine - ** is being called from within pager_playback(). The local value - ** of Pager.sectorSize is restored at the end of that routine. - */ - pPager->sectorSize = iSectorSize; - } - - pPager->journalOff += JOURNAL_HDR_SZ(pPager); - return rc; -} - - -/* -** Write the supplied master journal name into the journal file for pager -** pPager at the current location. The master journal name must be the last -** thing written to a journal file. If the pager is in full-sync mode, the -** journal file descriptor is advanced to the next sector boundary before -** anything is written. The format is: +** Within this comment block, a page is deemed to have been synced +** automatically as soon as it is written when PRAGMA synchronous=OFF. +** Otherwise, the page is not synced until the xSync method of the VFS +** is called successfully on the file containing the page. ** -** + 4 bytes: PAGER_MJ_PGNO. -** + N bytes: Master journal filename in utf-8. -** + 4 bytes: N (length of master journal name in bytes, no nul-terminator). -** + 4 bytes: Master journal name checksum. -** + 8 bytes: aJournalMagic[]. +** Definition: A page of the database file is said to be "overwriteable" if +** one or more of the following are true about the page: +** +** (a) The original content of the page as it was at the beginning of +** the transaction has been written into the rollback journal and +** synced. +** +** (b) The page was a freelist leaf page at the start of the transaction. +** +** (c) The page number is greater than the largest page that existed in +** the database file at the start of the transaction. +** +** (1) A page of the database file is never overwritten unless one of the +** following are true: +** +** (a) The page and all other pages on the same sector are overwriteable. +** +** (b) The atomic page write optimization is enabled, and the entire +** transaction other than the update of the transaction sequence +** number consists of a single page change. +** +** (2) The content of a page written into the rollback journal exactly matches +** both the content in the database when the rollback journal was written +** and the content in the database at the beginning of the current +** transaction. +** +** (3) Writes to the database file are an integer multiple of the page size +** in length and are aligned on a page boundary. +** +** (4) Reads from the database file are either aligned on a page boundary and +** an integer multiple of the page size in length or are taken from the +** first 100 bytes of the database file. +** +** (5) All writes to the database file are synced prior to the rollback journal +** being deleted, truncated, or zeroed. +** +** (6) If a master journal file is used, then all writes to the database file +** are synced prior to the master journal being deleted. +** +** Definition: Two databases (or the same database at two points it time) +** are said to be "logically equivalent" if they give the same answer to +** all queries. Note in particular the the content of freelist leaf +** pages can be changed arbitarily without effecting the logical equivalence +** of the database. +** +** (7) At any time, if any subset, including the empty set and the total set, +** of the unsynced changes to a rollback journal are removed and the +** journal is rolled back, the resulting database file will be logical +** equivalent to the database file at the beginning of the transaction. +** +** (8) When a transaction is rolled back, the xTruncate method of the VFS +** is called to restore the database file to the same size it was at +** the beginning of the transaction. (In some VFSes, the xTruncate +** method is a no-op, but that does not change the fact the SQLite will +** invoke it.) +** +** (9) Whenever the database file is modified, at least one bit in the range +** of bytes from 24 through 39 inclusive will be changed prior to releasing +** the EXCLUSIVE lock, thus signaling other connections on the same +** database to flush their caches. ** -** The master journal page checksum is the sum of the bytes in the master -** journal name, where each byte is interpreted as a signed 8-bit integer. +** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less +** than one billion transactions. ** -** If zMaster is a NULL pointer (occurs for a single database transaction), -** this call is a no-op. -*/ -static int writeMasterJournal(Pager *pPager, const char *zMaster){ - int rc; /* Return code */ - int nMaster; /* Length of string zMaster */ - i64 iHdrOff; /* Offset of header in journal file */ - i64 jrnlSize; /* Size of journal file on disk */ - u32 cksum = 0; /* Checksum of string zMaster */ - - assert( pPager->setMaster==0 ); - assert( !pagerUseWal(pPager) ); - - if( !zMaster - || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_OFF - ){ - return SQLITE_OK; - } - pPager->setMaster = 1; - assert( isOpen(pPager->jfd) ); - assert( pPager->journalHdr <= pPager->journalOff ); - - /* Calculate the length in bytes and the checksum of zMaster */ - for(nMaster=0; zMaster[nMaster]; nMaster++){ - cksum += zMaster[nMaster]; - } - - /* If in full-sync mode, advance to the next disk sector before writing - ** the master journal name. This is in case the previous page written to - ** the journal has already been synced. - */ - if( pPager->fullSync ){ - pPager->journalOff = journalHdrOffset(pPager); - } - iHdrOff = pPager->journalOff; - - /* Write the master journal data to the end of the journal file. If - ** an error occurs, return the error code to the caller. - */ - if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) - || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) - || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) - ){ - return rc; - } - pPager->journalOff += (nMaster+20); - - /* If the pager is in peristent-journal mode, then the physical - ** journal-file may extend past the end of the master-journal name - ** and 8 bytes of magic data just written to the file. This is - ** dangerous because the code to rollback a hot-journal file - ** will not be able to find the master-journal name to determine - ** whether or not the journal is hot. - ** - ** Easiest thing to do in this scenario is to truncate the journal - ** file to the required size. - */ - if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) - && jrnlSize>pPager->journalOff - ){ - rc = sqlite3OsTruncate(pPager->jfd, pPager->journalOff); - } - return rc; -} - -/* -** Find a page in the hash table given its page number. Return -** a pointer to the page or NULL if the requested page is not -** already in memory. -*/ -static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ - PgHdr *p; /* Return value */ - - /* It is not possible for a call to PcacheFetch() with createFlag==0 to - ** fail, since no attempt to allocate dynamic memory will be made. - */ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); - return p; -} - -/* -** Discard the entire contents of the in-memory page-cache. -*/ -static void pager_reset(Pager *pPager){ - sqlite3BackupRestart(pPager->pBackup); - sqlite3PcacheClear(pPager->pPCache); -} - -/* -** Free all structures in the Pager.aSavepoint[] array and set both -** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal -** if it is open and the pager is not in exclusive mode. -*/ -static void releaseAllSavepoints(Pager *pPager){ - int ii; /* Iterator for looping through Pager.aSavepoint */ - for(ii=0; iinSavepoint; ii++){ - sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); - } - if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ - sqlite3OsClose(pPager->sjfd); - } - sqlite3_free(pPager->aSavepoint); - pPager->aSavepoint = 0; - pPager->nSavepoint = 0; - pPager->nSubRec = 0; -} +** (11) A database file is well-formed at the beginning and at the conclusion +** of every transaction. +** +** (12) An EXCLUSIVE lock is held on the database file when writing to +** the database file. +** +** (13) A SHARED lock is held on the database file while reading any +** content out of the database file. +** +******************************************************************************/ /* -** Set the bit number pgno in the PagerSavepoint.pInSavepoint -** bitvecs of all open savepoints. Return SQLITE_OK if successful -** or SQLITE_NOMEM if a malloc failure occurs. +** Macros for troubleshooting. Normally turned off */ -static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ - int ii; /* Loop counter */ - int rc = SQLITE_OK; /* Result code */ - - for(ii=0; iinSavepoint; ii++){ - PagerSavepoint *p = &pPager->aSavepoint[ii]; - if( pgno<=p->nOrig ){ - rc |= sqlite3BitvecSet(p->pInSavepoint, pgno); - testcase( rc==SQLITE_NOMEM ); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - } - } - return rc; -} +#if 0 +int sqlite3PagerTrace=1; /* True to enable tracing */ +#define sqlite3DebugPrintf printf +#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; } +#else +#define PAGERTRACE(X) +#endif /* -** This function is a no-op if the pager is in exclusive mode and not -** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN -** state. -** -** If the pager is not in exclusive-access mode, the database file is -** completely unlocked. If the file is unlocked and the file-system does -** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is -** closed (if it is open). +** The following two macros are used within the PAGERTRACE() macros above +** to print out file-descriptors. ** -** If the pager is in ERROR state when this function is called, the -** contents of the pager cache are discarded before switching back to -** the OPEN state. Regardless of whether the pager is in exclusive-mode -** or not, any journal file left in the file-system will be treated -** as a hot-journal and rolled back the next time a read-transaction -** is opened (by this or by any other connection). +** PAGERID() takes a pointer to a Pager struct as its argument. The +** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file +** struct as its argument. */ -static void pager_unlock(Pager *pPager){ - - assert( pPager->eState==PAGER_READER - || pPager->eState==PAGER_OPEN - || pPager->eState==PAGER_ERROR - ); - - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - releaseAllSavepoints(pPager); - - if( pagerUseWal(pPager) ){ - assert( !isOpen(pPager->jfd) ); - sqlite3WalEndReadTransaction(pPager->pWal); - pPager->eState = PAGER_OPEN; - }else if( !pPager->exclusiveMode ){ - int rc; /* Error code returned by pagerUnlockDb() */ - int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; - - /* If the operating system support deletion of open files, then - ** close the journal file when dropping the database lock. Otherwise - ** another connection with journal_mode=delete might delete the file - ** out from under us. - */ - assert( (PAGER_JOURNALMODE_MEMORY & 5)!=1 ); - assert( (PAGER_JOURNALMODE_OFF & 5)!=1 ); - assert( (PAGER_JOURNALMODE_WAL & 5)!=1 ); - assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); - assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); - assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); - if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) - || 1!=(pPager->journalMode & 5) - ){ - sqlite3OsClose(pPager->jfd); - } - - /* If the pager is in the ERROR state and the call to unlock the database - ** file fails, set the current lock to UNKNOWN_LOCK. See the comment - ** above the #define for UNKNOWN_LOCK for an explanation of why this - ** is necessary. - */ - rc = pagerUnlockDb(pPager, NO_LOCK); - if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ - pPager->eLock = UNKNOWN_LOCK; - } - - /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here - ** without clearing the error code. This is intentional - the error - ** code is cleared and the cache reset in the block below. - */ - assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); - pPager->changeCountDone = 0; - pPager->eState = PAGER_OPEN; - } - - /* If Pager.errCode is set, the contents of the pager cache cannot be - ** trusted. Now that there are no outstanding references to the pager, - ** it can safely move back to PAGER_OPEN state. This happens in both - ** normal and exclusive-locking mode. - */ - if( pPager->errCode ){ - assert( !MEMDB ); - pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; - } - - pPager->journalOff = 0; - pPager->journalHdr = 0; - pPager->setMaster = 0; -} +#define PAGERID(p) ((int)(p->fd)) +#define FILEHANDLEID(fd) ((int)fd) /* -** This function is called whenever an IOERR or FULL error that requires -** the pager to transition into the ERROR state may ahve occurred. -** The first argument is a pointer to the pager structure, the second -** the error-code about to be returned by a pager API function. The -** value returned is a copy of the second argument to this function. +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the seven states shown in the following +** state diagram. ** -** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the -** IOERR sub-codes, the pager enters the ERROR state and the error code -** is stored in Pager.errCode. While the pager remains in the ERROR state, -** all major API calls on the Pager will immediately return Pager.errCode. +** OPEN <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ ** -** The ERROR state indicates that the contents of the pager-cache -** cannot be trusted. This state can be cleared by completely discarding -** the contents of the pager-cache. If a transaction was active when -** the persistent error occurred, then the rollback journal may need -** to be replayed to restore the contents of the database file (as if -** it were a hot-journal). -*/ -static int pager_error(Pager *pPager, int rc){ - int rc2 = rc & 0xff; - assert( rc==SQLITE_OK || !MEMDB ); - assert( - pPager->errCode==SQLITE_FULL || - pPager->errCode==SQLITE_OK || - (pPager->errCode & 0xff)==SQLITE_IOERR - ); - if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ - pPager->errCode = rc; - pPager->eState = PAGER_ERROR; - } - return rc; -} - -/* -** This routine ends a transaction. A transaction is usually ended by -** either a COMMIT or a ROLLBACK operation. This routine may be called -** after rollback of a hot-journal, or if an error occurs while opening -** the journal file or writing the very first journal-header of a -** database transaction. +** +** List of state transitions and the C [function] that performs each: ** -** This routine is never called in PAGER_ERROR state. If it is called -** in PAGER_NONE or PAGER_SHARED state and the lock held is less -** exclusive than a RESERVED lock, it is a no-op. +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] ** -** Otherwise, any active savepoints are released. +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] ** -** If the journal file is open, then it is "finalized". Once a journal -** file has been finalized it is not possible to use it to roll back a -** transaction. Nor will it be considered to be a hot-journal by this -** or any other database connection. Exactly how a journal is finalized -** depends on whether or not the pager is running in exclusive mode and -** the current journal-mode (Pager.journalMode value), as follows: +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** ** -** journalMode==MEMORY -** Journal file descriptor is simply closed. This destroys an -** in-memory journal. +** OPEN: ** -** journalMode==TRUNCATE -** Journal file is truncated to zero bytes in size. +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. ** -** journalMode==PERSIST -** The first 28 bytes of the journal file are zeroed. This invalidates -** the first journal header in the file, and hence the entire journal -** file. An invalid journal file cannot be rolled back. +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. ** -** journalMode==DELETE -** The journal file is closed and deleted using sqlite3OsDelete(). +** READER: ** -** If the pager is running in exclusive mode, this method of finalizing -** the journal file is never used. Instead, if the journalMode is -** DELETE and the pager is in exclusive mode, the method described under -** journalMode==PERSIST is used instead. +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. ** -** After the journal is finalized, the pager moves to PAGER_READER state. -** If running in non-exclusive rollback mode, the lock on the file is -** downgraded to a SHARED_LOCK. +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. ** -** SQLITE_OK is returned if no error occurs. If an error occurs during -** any of the IO operations to finalize the journal file or unlock the -** database then the IO error code is returned to the user. If the -** operation to finalize the journal file fails, then the code still -** tries to unlock the database file if not in exclusive mode. If the -** unlock operation fails as well, then the first error code related -** to the first error encountered (the journal finalization one) is -** returned. -*/ -static int pager_end_transaction(Pager *pPager, int hasMaster){ - int rc = SQLITE_OK; /* Error code from journal finalization operation */ - int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ - - /* Do nothing if the pager does not have an open write transaction - ** or at least a RESERVED lock. This function may be called when there - ** is no write-transaction active but a RESERVED or greater lock is - ** held under two circumstances: - ** - ** 1. After a successful hot-journal rollback, it is called with - ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. - ** - ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE - ** lock switches back to locking_mode=normal and then executes a - ** read-transaction, this function is called with eState==PAGER_READER - ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. - */ - assert( assert_pager_state(pPager) ); - assert( pPager->eState!=PAGER_ERROR ); - if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); - if( isOpen(pPager->jfd) ){ - assert( !pagerUseWal(pPager) ); - - /* Finalize the journal file. */ - if( sqlite3IsMemJournal(pPager->jfd) ){ - assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); - sqlite3OsClose(pPager->jfd); - }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ - if( pPager->journalOff==0 ){ - rc = SQLITE_OK; - }else{ - rc = sqlite3OsTruncate(pPager->jfd, 0); - } - pPager->journalOff = 0; - }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST - || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) - ){ - rc = zeroJournalHdr(pPager, hasMaster); - pPager->journalOff = 0; - }else{ - /* This branch may be executed with Pager.journalMode==MEMORY if - ** a hot-journal was just rolled back. In this case the journal - ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. - */ - assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE - || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_WAL - ); - sqlite3OsClose(pPager->jfd); - if( !pPager->tempFile ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); - } - } - } - -#ifdef SQLITE_CHECK_PAGES - sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); - if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ - PgHdr *p = pager_lookup(pPager, 1); - if( p ){ - p->pageHash = 0; - sqlite3PagerUnref(p); - } - } -#endif - - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); - - if( pagerUseWal(pPager) ){ - /* Drop the WAL write-lock, if any. Also, if the connection was in - ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE - ** lock held on the database file. - */ - rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); - assert( rc2==SQLITE_OK ); - } - if( !pPager->exclusiveMode - && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) - ){ - rc2 = pagerUnlockDb(pPager, SHARED_LOCK); - pPager->changeCountDone = 0; - } - pPager->eState = PAGER_READER; - pPager->setMaster = 0; - - return (rc==SQLITE_OK?rc2:rc); -} - -/* -** Execute a rollback if a transaction is active and unlock the -** database file. +** WRITER_LOCKED: ** -** If the pager has already entered the ERROR state, do not attempt -** the rollback at this time. Instead, pager_unlock() is called. The -** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and move the pager back to OPEN state. If this -** means that there is a hot-journal left in the file-system, the next -** connection to obtain a shared lock on the pager (which may be this one) -** will roll it back. +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. ** -** If the pager has not already entered the ERROR state, but an IO or -** malloc error occurs during a rollback, then this will itself cause -** the pager to enter the ERROR state. Which will be cleared by the -** call to pager_unlock(), as described above. -*/ -static void pagerUnlockAndRollback(Pager *pPager){ - if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ - assert( assert_pager_state(pPager) ); - if( pPager->eState>=PAGER_WRITER_LOCKED ){ - sqlite3BeginBenignMalloc(); - sqlite3PagerRollback(pPager); - sqlite3EndBenignMalloc(); - }else if( !pPager->exclusiveMode ){ - assert( pPager->eState==PAGER_READER ); - pager_end_transaction(pPager, 0); - } - } - pager_unlock(pPager); -} - -/* -** Parameter aData must point to a buffer of pPager->pageSize bytes -** of data. Compute and return a checksum based ont the contents of the -** page of data and the current value of pPager->cksumInit. +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. ** -** This is not a real checksum. It is really just the sum of the -** random initial value (pPager->cksumInit) and every 200th byte -** of the page data, starting with byte offset (pPager->pageSize%200). -** Each byte is interpreted as an 8-bit unsigned integer. +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. ** -** Changing the formula used to compute this checksum results in an -** incompatible journal file format. +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. ** -** If journal corruption occurs due to a power failure, the most likely -** scenario is that one end or the other of the record will be changed. -** It is much less likely that the two ends of the journal record will be -** correct and the middle be corrupt. Thus, this "checksum" scheme, -** though fast and simple, catches the mostly likely kind of corruption. +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occured, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). */ -static u32 pager_cksum(Pager *pPager, const u8 *aData){ - u32 cksum = pPager->cksumInit; /* Checksum value to return */ - int i = pPager->pageSize-200; /* Loop counter */ - while( i>0 ){ - cksum += aData[i]; - i -= 200; - } - return cksum; -} +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 /* -** Report the current page size and number of reserved bytes back -** to the codec. +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + +/* +** A macro used for invoking the codec if there is one */ #ifdef SQLITE_HAS_CODEC -static void pagerReportSize(Pager *pPager){ - if( pPager->xCodecSizeChng ){ - pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, - (int)pPager->nReserve); - } -} +# define CODEC1(P,D,N,X,E) \ + if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } +# define CODEC2(P,D,N,X,E,O) \ + if( P->xCodec==0 ){ O=(char*)D; }else \ + if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } #else -# define pagerReportSize(X) /* No-op if we do not support a codec */ +# define CODEC1(P,D,N,X,E) /* NO-OP */ +# define CODEC2(P,D,N,X,E,O) O=(char*)D #endif /* -** Read a single page from either the journal file (if isMainJrnl==1) or -** from the sub-journal (if isMainJrnl==0) and playback that page. -** The page begins at offset *pOffset into the file. The *pOffset -** value is increased to the start of the next page in the journal. +** The maximum allowed sector size. 64KiB. If the xSectorsize() method +** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +** This could conceivably cause corruption following a power failure on +** such a system. This is currently an undocumented limit. +*/ +#define MAX_SECTOR_SIZE 0x10000 + +/* +** An instance of the following structure is allocated for each active +** savepoint and statement transaction in the system. All such structures +** are stored in the Pager.aSavepoint[] array, which is allocated and +** resized using sqlite3Realloc(). ** -** The main rollback journal uses checksums - the statement journal does -** not. +** When a savepoint is created, the PagerSavepoint.iHdrOffset field is +** set to 0. If a journal-header is written into the main journal while +** the savepoint is active, then iHdrOffset is set to the byte offset +** immediately following the last journal record written into the main +** journal before the journal-header. This is required during savepoint +** rollback (see pagerPlaybackSavepoint()). +*/ +typedef struct PagerSavepoint PagerSavepoint; +struct PagerSavepoint { + i64 iOffset; /* Starting offset in main journal */ + i64 iHdrOffset; /* See above */ + Bitvec *pInSavepoint; /* Set of pages in this savepoint */ + Pgno nOrig; /* Original number of pages in file */ + Pgno iSubRec; /* Index of first record in sub-journal */ +#ifndef SQLITE_OMIT_WAL + u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ +#endif +}; + +/* +** A open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: ** -** If the page number of the page record read from the (sub-)journal file -** is greater than the current value of Pager.dbSize, then playback is -** skipped and SQLITE_OK is returned. +** eState ** -** If pDone is not NULL, then it is a record of pages that have already -** been played back. If the page at *pOffset has already been played back -** (if the corresponding pDone bit is set) then skip the playback. -** Make sure the pDone bit corresponding to the *pOffset page is set -** prior to returning. +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. ** -** If the page record is successfully read from the (sub-)journal file -** and played back, then SQLITE_OK is returned. If an IO error occurs -** while reading the record from the (sub-)journal file or while writing -** to the database file, then the IO error code is returned. If data -** is successfully read from the (sub-)journal file but appears to be -** corrupted, SQLITE_DONE is returned. Data is considered corrupted in -** two circumstances: +** eLock +** +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. +** +** changeCountDone +** +** This boolean variable is used to make sure that the change-counter +** (the 4-byte header field at byte offset 24 of the database file) is +** not updated more often than necessary. +** +** It is set to true when the change-counter field is updated, which +** can only happen if an exclusive lock is held on the database file. +** It is cleared (set to false) whenever an exclusive lock is +** relinquished on the database file. Each time a transaction is committed, +** The changeCountDone flag is inspected. If it is true, the work of +** updating the change-counter is omitted for the current transaction. +** +** This mechanism means that when running in exclusive mode, a connection +** need only update the change-counter once, for the first transaction +** committed. +** +** setMaster +** +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a master-journal name to be written into the +** journal file before it is synced to disk. +** +** Whether or not a journal file contains a master-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a master-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a master-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain master journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** master journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setMaster flag +** is cleared anyway (and the pager will move to ERROR state). +** +** doNotSpill, doNotSyncSpill +** +** These two boolean variables control the behaviour of cache-spills +** (calls made by the pcache module to the pagerStress() routine to +** write cached data to the file-system in order to free up memory). +** +** When doNotSpill is non-zero, writing to the database from pagerStress() +** is disabled altogether. This is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). ** -** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or -** * If the record is being rolled back from the main journal file -** and the checksum field does not match the record content. +** If doNotSyncSpill is non-zero, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. ** -** Neither of these two scenarios are possible during a savepoint rollback. +** subjInMemory ** -** If this is a savepoint rollback, then memory may have to be dynamically -** allocated by this function. If this is the case and an allocation fails, -** SQLITE_NOMEM is returned. +** This is a boolean variable. If true, then any required sub-journal +** is opened as an in-memory journal file. If false, then in-memory +** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. */ -static int pager_playback_one_page( - Pager *pPager, /* The pager being played back */ - i64 *pOffset, /* Offset of record to playback */ - Bitvec *pDone, /* Bitvec of pages already played back */ - int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */ - int isSavepnt /* True for a savepoint rollback */ -){ - int rc; - PgHdr *pPg; /* An existing page in the cache */ - Pgno pgno; /* The page number of a page in journal */ - u32 cksum; /* Checksum used for sanity checking */ - char *aData; /* Temporary storage for the page */ - sqlite3_file *jfd; /* The file descriptor for the journal file */ - int isSynced; /* True if journal page is synced */ - - assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ - assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ - assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ - assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ - - aData = pPager->pTmpSpace; - assert( aData ); /* Temp storage must have already been allocated */ - assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); - - /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction - ** or savepoint rollback done at the request of the caller) or this is - ** a hot-journal rollback. If it is a hot-journal rollback, the pager - ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback - ** only reads from the main journal, not the sub-journal. - */ - assert( pPager->eState>=PAGER_WRITER_CACHEMOD - || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) - ); - assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); +struct Pager { + sqlite3_vfs *pVfs; /* OS functions to use for IO */ + u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ + u8 useJournal; /* Use a rollback journal on this file */ + u8 noReadlock; /* Do not bother to obtain readlocks */ + u8 noSync; /* Do not sync the journal if true */ + u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ + u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 tempFile; /* zFilename is a temporary file */ + u8 readOnly; /* True for a read-only database */ + u8 memDb; /* True to inhibit all file I/O */ - /* Read the page number and page data from the journal or sub-journal - ** file. Return an error code to the caller if an IO error occurs. + /************************************************************************** + ** The following block contains those class members that change during + ** routine opertion. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. */ - jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; - rc = read32bits(jfd, *pOffset, &pgno); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4); - if( rc!=SQLITE_OK ) return rc; - *pOffset += pPager->pageSize + 4 + isMainJrnl*4; + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ + u8 changeCountDone; /* Set after incrementing the change-counter */ + u8 setMaster; /* True if a m-j name has been written to jrnl */ + u8 doNotSpill; /* Do not spill the cache when non-zero */ + u8 doNotSyncSpill; /* Do not do a spill that requires jrnl sync */ + u8 subjInMemory; /* True to use in-memory sub-journals */ + Pgno dbSize; /* Number of pages in the database */ + Pgno dbOrigSize; /* dbSize before the current transaction */ + Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ + int errCode; /* One of several kinds of errors */ + int nRec; /* Pages journalled since last j-header written */ + u32 cksumInit; /* Quasi-random value added to every checksum */ + u32 nSubRec; /* Number of records written to sub-journal */ + Bitvec *pInJournal; /* One bit for each page in the database file */ + sqlite3_file *fd; /* File descriptor for database */ + sqlite3_file *jfd; /* File descriptor for main journal */ + sqlite3_file *sjfd; /* File descriptor for sub-journal */ + i64 journalOff; /* Current write offset in the journal file */ + i64 journalHdr; /* Byte offset to previous journal header */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ + PagerSavepoint *aSavepoint; /* Array of active savepoints */ + int nSavepoint; /* Number of elements in aSavepoint[] */ + char dbFileVers[16]; /* Changes whenever database file changes */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ - /* Sanity checking on the page. This is more important that I originally - ** thought. If a power failure occurs while the journal is being written, - ** it could cause invalid data to be written into the journal. We need to - ** detect this invalid data (with high probability) and ignore it. - */ - if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ - assert( !isSavepnt ); - return SQLITE_DONE; - } - if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){ - return SQLITE_OK; - } - if( isMainJrnl ){ - rc = read32bits(jfd, (*pOffset)-4, &cksum); - if( rc ) return rc; - if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){ - return SQLITE_DONE; - } - } - - /* If this page has already been played by before during the current - ** rollback, then don't bother to play it back again. - */ - if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ - return rc; - } - - /* When playing back page 1, restore the nReserve setting - */ - if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ - pPager->nReserve = ((u8*)aData)[20]; - pagerReportSize(pPager); - } + u16 nExtra; /* Add this many bytes to each in-memory page */ + i16 nReserve; /* Number of unused bytes at end of each page */ + u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ + int pageSize; /* Number of bytes in a page */ + Pgno mxPgno; /* Maximum allowed size of the database */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ + char *zFilename; /* Name of the database file */ + char *zJournal; /* Name of the journal file */ + int (*xBusyHandler)(void*); /* Function to call when busy */ + void *pBusyHandlerArg; /* Context argument for xBusyHandler */ +#ifdef SQLITE_TEST + int nHit, nMiss; /* Cache hits and missing */ + int nRead, nWrite; /* Database pages read/written */ +#endif + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ +#ifdef SQLITE_HAS_CODEC + void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ + void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ + void (*xCodecFree)(void*); /* Destructor for the codec */ + void *pCodec; /* First argument to xCodec... methods */ +#endif + char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ + PCache *pPCache; /* Pointer to page cache object */ +#ifndef SQLITE_OMIT_WAL + Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ + char *zWal; /* File name for write-ahead log */ +#endif +}; - /* If the pager is in CACHEMOD state, then there must be a copy of this - ** page in the pager cache. In this case just update the pager cache, - ** not the database file. The page is left marked dirty in this case. - ** - ** An exception to the above rule: If the database is in no-sync mode - ** and a page is moved during an incremental vacuum then the page may - ** not be in the pager cache. Later: if a malloc() or IO error occurs - ** during a Movepage() call, then the page may not be in the cache - ** either. So the condition described in the above paragraph is not - ** assert()able. - ** - ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the - ** pager cache if it exists and the main file. The page is then marked - ** not dirty. Since this code is only executed in PAGER_OPEN state for - ** a hot-journal rollback, it is guaranteed that the page-cache is empty - ** if the pager is in OPEN state. - ** - ** Ticket #1171: The statement journal might contain page content that is - ** different from the page content at the start of the transaction. - ** This occurs when a page is changed prior to the start of a statement - ** then changed again within the statement. When rolling back such a - ** statement we must not write to the original database unless we know - ** for certain that original page contents are synced into the main rollback - ** journal. Otherwise, a power loss might leave modified data in the - ** database file without an entry in the rollback journal that can - ** restore the database to its original form. Two conditions must be - ** met before writing to the database files. (1) the database must be - ** locked. (2) we know that the original page content is fully synced - ** in the main journal either because the page is not in cache or else - ** the page is marked as needSync==0. - ** - ** 2008-04-14: When attempting to vacuum a corrupt database file, it - ** is possible to fail a statement on a database that does not yet exist. - ** Do not attempt to write if database file has never been opened. - */ - if( pagerUseWal(pPager) ){ - pPg = 0; - }else{ - pPg = pager_lookup(pPager, pgno); - } - assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); - PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", - PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), - (isMainJrnl?"main-journal":"sub-journal") - )); - if( isMainJrnl ){ - isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr); - }else{ - isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); - } - if( isOpen(pPager->fd) - && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) - && isSynced - ){ - i64 ofst = (pgno-1)*(i64)pPager->pageSize; - testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); - assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); - if( pgno>pPager->dbFileSize ){ - pPager->dbFileSize = pgno; - } - if( pPager->pBackup ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); - sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); - } - }else if( !isMainJrnl && pPg==0 ){ - /* If this is a rollback of a savepoint and data was not written to - ** the database and the page is not in-memory, there is a potential - ** problem. When the page is next fetched by the b-tree layer, it - ** will be read from the database file, which may or may not be - ** current. - ** - ** There are a couple of different ways this can happen. All are quite - ** obscure. When running in synchronous mode, this can only happen - ** if the page is on the free-list at the start of the transaction, then - ** populated, then moved using sqlite3PagerMovepage(). - ** - ** The solution is to add an in-memory page to the cache containing - ** the data just read from the sub-journal. Mark the page as dirty - ** and if the pager requires a journal-sync, then mark the page as - ** requiring a journal-sync before it is written. - */ - assert( isSavepnt ); - assert( pPager->doNotSpill==0 ); - pPager->doNotSpill++; - rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1); - assert( pPager->doNotSpill==1 ); - pPager->doNotSpill--; - if( rc!=SQLITE_OK ) return rc; - pPg->flags &= ~PGHDR_NEED_READ; - sqlite3PcacheMakeDirty(pPg); - } - if( pPg ){ - /* No page should ever be explicitly rolled back that is in use, except - ** for page 1 which is held in use in order to keep the lock on the - ** database active. However such a page may be rolled back as a result - ** of an internal error resulting in an automatic call to - ** sqlite3PagerRollback(). - */ - void *pData; - pData = pPg->pData; - memcpy(pData, (u8*)aData, pPager->pageSize); - pPager->xReiniter(pPg); - if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ - /* If the contents of this page were just restored from the main - ** journal file, then its content must be as they were when the - ** transaction was first opened. In this case we can mark the page - ** as clean, since there will be no need to write it out to the - ** database. - ** - ** There is one exception to this rule. If the page is being rolled - ** back as part of a savepoint (or statement) rollback from an - ** unsynced portion of the main journal file, then it is not safe - ** to mark the page as clean. This is because marking the page as - ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is - ** already in the journal file (recorded in Pager.pInJournal) and - ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to - ** again within this transaction, it will be marked as dirty but - ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially - ** be written out into the database file before its journal file - ** segment is synced. If a crash occurs during or following this, - ** database corruption may ensue. - */ - assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); - } - pager_set_pagehash(pPg); +/* +** The following global variables hold counters used for +** testing purposes only. These variables do not exist in +** a non-testing build. These variables are not thread-safe. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ +SQLITE_API int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ +SQLITE_API int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ +# define PAGER_INCR(v) v++ +#else +# define PAGER_INCR(v) +#endif - /* If this was page 1, then restore the value of Pager.dbFileVers. - ** Do this before any decoding. */ - if( pgno==1 ){ - memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); - } - /* Decode the page just read from disk */ - CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); - sqlite3PcacheRelease(pPg); - } - return rc; -} /* -** Parameter zMaster is the name of a master journal file. A single journal -** file that referred to the master journal file has just been rolled back. -** This routine checks if it is possible to delete the master journal file, -** and does so if it is. -** -** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not -** available for use within this function. +** Journal files begin with the following magic string. The data +** was obtained from /dev/random. It is used only as a sanity check. ** -** When a master journal file is created, it is populated with the names -** of all of its child journals, one after another, formatted as utf-8 -** encoded text. The end of each child journal file is marked with a -** nul-terminator byte (0x00). i.e. the entire contents of a master journal -** file for a transaction involving two databases might be: +** Since version 2.8.0, the journal format contains additional sanity +** checking information. If the power fails while the journal is being +** written, semi-random garbage data might appear in the journal +** file after power is restored. If an attempt is then made +** to roll the journal back, the database could be corrupted. The additional +** sanity checking data is an attempt to discover the garbage in the +** journal and ignore it. ** -** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" +** The sanity checking information for the new journal format consists +** of a 32-bit checksum on each page of data. The checksum covers both +** the page number and the pPager->pageSize bytes of data for the page. +** This cksum is initialized to a 32-bit random value that appears in the +** journal file right after the header. The random initializer is important, +** because garbage data that appears at the end of a journal is likely +** data that was once in other files that have now been deleted. If the +** garbage data came from an obsolete journal file, the checksums might +** be correct. But by initializing the checksum to random value which +** is different for every journal, we minimize that risk. +*/ +static const unsigned char aJournalMagic[] = { + 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7, +}; + +/* +** The size of the of each page record in the journal is given by +** the following macro. +*/ +#define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) + +/* +** The journal header size for this pager. This is usually the same +** size as a single disk sector. See also setSectorSize(). +*/ +#define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) + +/* +** The macro MEMDB is true if we are dealing with an in-memory database. +** We do this as a macro so that if the SQLITE_OMIT_MEMORYDB macro is set, +** the value of MEMDB will be a constant and the compiler will optimize +** out code that would never execute. +*/ +#ifdef SQLITE_OMIT_MEMORYDB +# define MEMDB 0 +#else +# define MEMDB pPager->memDb +#endif + +/* +** The maximum legal page number is (2^31 - 1). +*/ +#define PAGER_MAX_PGNO 2147483647 + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. ** -** A master journal file may only be deleted once all of its child -** journals have been rolled back. +** This is so that expressions can be written as: ** -** This function reads the contents of the master-journal file into -** memory and loops through each of the child journal names. For -** each child journal, it checks if: +** if( isOpen(pPager->jfd) ){ ... ** -** * if the child journal exists, and if so -** * if the child journal contains a reference to master journal -** file zMaster +** instead of ** -** If a child journal can be found that matches both of the criteria -** above, this function returns without doing anything. Otherwise, if -** no such child journal can be found, file zMaster is deleted from -** the file-system using sqlite3OsDelete(). +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods) + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +#ifndef SQLITE_OMIT_WAL +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); +} +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + +#ifndef NDEBUG +/* +** Usage: ** -** If an IO error within this function, an error code is returned. This -** function allocates memory by calling sqlite3Malloc(). If an allocation -** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors -** occur, SQLITE_OK is returned. +** assert( assert_pager_state(pPager) ); ** -** TODO: This function allocates a single block of memory to load -** the entire contents of the master journal file. This could be -** a couple of kilobytes or so - potentially larger than the page -** size. +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. */ -static int pager_delmaster(Pager *pPager, const char *zMaster){ - sqlite3_vfs *pVfs = pPager->pVfs; - int rc; /* Return code */ - sqlite3_file *pMaster; /* Malloc'd master-journal file descriptor */ - sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */ - char *zMasterJournal = 0; /* Contents of master journal file */ - i64 nMasterJournal; /* Size of master journal file */ - char *zJournal; /* Pointer to one journal within MJ file */ - char *zMasterPtr; /* Space to hold MJ filename from a journal file */ - int nMasterPtr; /* Amount of space allocated to zMasterPtr[] */ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; - /* Allocate space for both the pJournal and pMaster file descriptors. - ** If successful, open the master journal file for reading. + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); + + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. */ - pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); - pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); - if( !pMaster ){ - rc = SQLITE_NOMEM; - }else{ - const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); - rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); - } - if( rc!=SQLITE_OK ) goto delmaster_out; + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); - /* Load the entire master journal file into space obtained from - ** sqlite3_malloc() and pointed to by zMasterJournal. Also obtain - ** sufficient space (in zMasterPtr) to hold the names of master - ** journal files extracted from regular rollback-journals. + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. */ - rc = sqlite3OsFileSize(pMaster, &nMasterJournal); - if( rc!=SQLITE_OK ) goto delmaster_out; - nMasterPtr = pVfs->mxPathname+1; - zMasterJournal = sqlite3Malloc((int)nMasterJournal + nMasterPtr + 1); - if( !zMasterJournal ){ - rc = SQLITE_NOMEM; - goto delmaster_out; + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); } - zMasterPtr = &zMasterJournal[nMasterJournal+1]; - rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); - if( rc!=SQLITE_OK ) goto delmaster_out; - zMasterJournal[nMasterJournal] = 0; - zJournal = zMasterJournal; - while( (zJournal-zMasterJournal)changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); - rc = readMasterJournal(pJournal, zMasterPtr, nMasterPtr); - sqlite3OsClose(pJournal); - if( rc!=SQLITE_OK ){ - goto delmaster_out; + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK || p->noReadlock ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setMaster==0 ); + break; - c = zMasterPtr[0]!=0 && strcmp(zMasterPtr, zMaster)==0; - if( c ){ - /* We have a match. Do not delete the master journal file. */ - goto delmaster_out; + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); } - } - zJournal += (sqlite3Strlen30(zJournal)+1); - } - - sqlite3OsClose(pMaster); - rc = sqlite3OsDelete(pVfs, zMaster, 0); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; -delmaster_out: - sqlite3_free(zMasterJournal); - if( pMaster ){ - sqlite3OsClose(pMaster); - assert( !isOpen(pJournal) ); - sqlite3_free(pMaster); - } - return rc; -} + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; -/* -** This function is used to change the actual size of the database -** file in the file-system. This only happens when committing a transaction, -** or rolling back a transaction (including rolling back a hot-journal). -** -** If the main database file is not open, or the pager is not in either -** DBMOD or OPEN state, this function is a no-op. Otherwise, the size -** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). -** If the file on disk is currently larger than nPage pages, then use the VFS -** xTruncate() method to truncate it. -** -** Or, it might might be the case that the file on disk is smaller than -** nPage pages. Some operating system implementations can get confused if -** you try to truncate a file to some size that is larger than it -** currently is, so detect this case and write a single zero byte to -** the end of the new file instead. -** -** If successful, return SQLITE_OK. If an IO error occurs while modifying -** the database file, return the error code to the caller. -*/ -static int pager_truncate(Pager *pPager, Pgno nPage){ - int rc = SQLITE_OK; - assert( pPager->eState!=PAGER_ERROR ); - assert( pPager->eState!=PAGER_READER ); - - if( isOpen(pPager->fd) - && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) - ){ - i64 currentSize, newSize; - assert( pPager->eLock==EXCLUSIVE_LOCK ); - /* TODO: Is it safe to use Pager.dbFileSize here? */ - rc = sqlite3OsFileSize(pPager->fd, ¤tSize); - newSize = pPager->pageSize*(i64)nPage; - if( rc==SQLITE_OK && currentSize!=newSize ){ - if( currentSize>newSize ){ - rc = sqlite3OsTruncate(pPager->fd, newSize); - }else{ - rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1); - } - if( rc==SQLITE_OK ){ - pPager->dbFileSize = nPage; - } - } + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + break; } - return rc; + + return 1; } +#endif /* ifndef NDEBUG */ +#ifdef SQLITE_DEBUG /* -** Set the value of the Pager.sectorSize variable for the given -** pager based on the value returned by the xSectorSize method -** of the open database file. The sector size will be used used -** to determine the size and alignment of journal header and -** master journal pointers within created journal files. -** -** For temporary files the effective sector size is always 512 bytes. +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: ** -** Otherwise, for non-temporary files, the effective sector size is -** the value returned by the xSectorSize() method rounded up to 32 if -** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it -** is greater than MAX_SECTOR_SIZE. +** (gdb) printf "%s", print_pager_state(pPager) */ -static void setSectorSize(Pager *pPager){ - assert( isOpen(pPager->fd) || pPager->tempFile ); +static char *print_pager_state(Pager *p){ + static char zRet[1024]; - if( !pPager->tempFile ){ - /* Sector size doesn't matter for temporary files. Also, the file - ** may not have been opened yet, in which case the OsSectorSize() - ** call will segfault. - */ - pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); - } - if( pPager->sectorSize<32 ){ - pPager->sectorSize = 512; - } - if( pPager->sectorSize>MAX_SECTOR_SIZE ){ - assert( MAX_SECTOR_SIZE>=512 ); - pPager->sectorSize = MAX_SECTOR_SIZE; - } + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; } +#endif /* -** Playback the journal and thus restore the database file to -** the state it was in before we started making changes. -** -** The journal file format is as follows: -** -** (1) 8 byte prefix. A copy of aJournalMagic[]. -** (2) 4 byte big-endian integer which is the number of valid page records -** in the journal. If this value is 0xffffffff, then compute the -** number of page records from the journal size. -** (3) 4 byte big-endian integer which is the initial value for the -** sanity checksum. -** (4) 4 byte integer which is the number of pages to truncate the -** database to during a rollback. -** (5) 4 byte big-endian integer which is the sector size. The header -** is this many bytes in size. -** (6) 4 byte big-endian integer which is the page size. -** (7) zero padding out to the next sector size. -** (8) Zero or more pages instances, each as follows: -** + 4 byte page number. -** + pPager->pageSize bytes of data. -** + 4 byte checksum -** -** When we speak of the journal header, we mean the first 7 items above. -** Each entry in the journal is an instance of the 8th item. -** -** Call the value from the second bullet "nRec". nRec is the number of -** valid page entries in the journal. In most cases, you can compute the -** value of nRec from the size of the journal file. But if a power -** failure occurred while the journal was being written, it could be the -** case that the size of the journal file had already been increased but -** the extra entries had not yet made it safely to disk. In such a case, -** the value of nRec computed from the file size would be too large. For -** that reason, we always use the nRec value in the header. -** -** If the nRec value is 0xffffffff it means that nRec should be computed -** from the file size. This value is used when the user selects the -** no-sync option for the journal. A power failure could lead to corruption -** in this case. But for things like temporary table (which will be -** deleted when the power is restored) we don't care. -** -** If the file opened as the journal file is not a well-formed -** journal file then all pages up to the first corrupted page are rolled -** back (or no pages if the journal header is corrupted). The journal file -** is then deleted and SQLITE_OK returned, just as if no corruption had -** been encountered. -** -** If an I/O or malloc() error occurs, the journal-file is not deleted -** and an error code is returned. +** Return true if it is necessary to write page *pPg into the sub-journal. +** A page needs to be written into the sub-journal if there exists one +** or more open savepoints for which: ** -** The isHot parameter indicates that we are trying to rollback a journal -** that might be a hot journal. Or, it could be that the journal is -** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. -** If the journal really is hot, reset the pager cache prior rolling -** back any content. If the journal is merely persistent, no reset is -** needed. +** * The page-number is less than or equal to PagerSavepoint.nOrig, and +** * The bit corresponding to the page-number is not set in +** PagerSavepoint.pInSavepoint. */ -static int pager_playback(Pager *pPager, int isHot){ - sqlite3_vfs *pVfs = pPager->pVfs; - i64 szJ; /* Size of the journal file in bytes */ - u32 nRec; /* Number of Records in the journal */ - u32 u; /* Unsigned loop counter */ - Pgno mxPg = 0; /* Size of the original file in pages */ - int rc; /* Result code of a subroutine */ - int res = 1; /* Value returned by sqlite3OsAccess() */ - char *zMaster = 0; /* Name of master journal file if any */ - int needPagerReset; /* True to reset page prior to first page rollback */ - - /* Figure out how many records are in the journal. Abort early if - ** the journal is empty. - */ - assert( isOpen(pPager->jfd) ); - rc = sqlite3OsFileSize(pPager->jfd, &szJ); - if( rc!=SQLITE_OK ){ - goto end_playback; - } - - /* Read the master journal name from the journal, if it is present. - ** If a master journal file name is specified, but the file is not - ** present on disk, then the journal is not hot and does not need to be - ** played back. - ** - ** TODO: Technically the following is an error because it assumes that - ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that - ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, - ** mxPathname is 512, which is the same as the minimum allowable value - ** for pageSize. - */ - zMaster = pPager->pTmpSpace; - rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); - if( rc==SQLITE_OK && zMaster[0] ){ - rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); - } - zMaster = 0; - if( rc!=SQLITE_OK || !res ){ - goto end_playback; - } - pPager->journalOff = 0; - needPagerReset = isHot; - - /* This loop terminates either when a readJournalHdr() or - ** pager_playback_one_page() call returns SQLITE_DONE or an IO error - ** occurs. - */ - while( 1 ){ - /* Read the next journal header from the journal file. If there are - ** not enough bytes left in the journal file for a complete header, or - ** it is corrupted, then a process must have failed while writing it. - ** This indicates nothing more needs to be rolled back. - */ - rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - } - goto end_playback; - } - - /* If nRec is 0xffffffff, then this journal was created by a process - ** working in no-sync mode. This means that the rest of the journal - ** file consists of pages, there are no more journal headers. Compute - ** the value of nRec based on this assumption. - */ - if( nRec==0xffffffff ){ - assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ); - nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager)); - } - - /* If nRec is 0 and this rollback is of a transaction created by this - ** process and if this is the final header in the journal, then it means - ** that this part of the journal was being filled but has not yet been - ** synced to disk. Compute the number of pages based on the remaining - ** size of the file. - ** - ** The third term of the test was added to fix ticket #2565. - ** When rolling back a hot journal, nRec==0 always means that the next - ** chunk of the journal contains zero pages to be rolled back. But - ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in - ** the journal, it means that the journal might contain additional - ** pages that need to be rolled back and that the number of pages - ** should be computed based on the journal file size. - */ - if( nRec==0 && !isHot && - pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ - nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager)); - } - - /* If this is the first header read from the journal, truncate the - ** database file back to its original size. - */ - if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ - rc = pager_truncate(pPager, mxPg); - if( rc!=SQLITE_OK ){ - goto end_playback; - } - pPager->dbSize = mxPg; - } - - /* Copy original pages out of the journal and back into the - ** database file and/or page cache. - */ - for(u=0; ujournalOff,0,1,0); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - pPager->journalOff = szJ; - break; - }else if( rc==SQLITE_IOERR_SHORT_READ ){ - /* If the journal has been truncated, simply stop reading and - ** processing the journal. This might happen if the journal was - ** not completely written and synced prior to a crash. In that - ** case, the database should have never been written in the - ** first place so it is OK to simply abandon the rollback. */ - rc = SQLITE_OK; - goto end_playback; - }else{ - /* If we are unable to rollback, quit and return the error - ** code. This will cause the pager to enter the error state - ** so that no further harm will be done. Perhaps the next - ** process to come along will be able to rollback the database. - */ - goto end_playback; - } - } +static int subjRequiresPage(PgHdr *pPg){ + Pgno pgno = pPg->pgno; + Pager *pPager = pPg->pPager; + int i; + for(i=0; inSavepoint; i++){ + PagerSavepoint *p = &pPager->aSavepoint[i]; + if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + return 1; } } - /*NOTREACHED*/ - assert( 0 ); - -end_playback: - /* Following a rollback, the database file should be back in its original - ** state prior to the start of the transaction, so invoke the - ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the - ** assertion that the transaction counter was modified. - */ - assert( - pPager->fd->pMethods==0 || - sqlite3OsFileControl(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0)>=SQLITE_OK - ); + return 0; +} - /* If this playback is happening automatically as a result of an IO or - ** malloc error that occurred after the change-counter was updated but - ** before the transaction was committed, then the change-counter - ** modification may just have been reverted. If this happens in exclusive - ** mode, then subsequent transactions performed by the connection will not - ** update the change-counter at all. This may lead to cache inconsistency - ** problems for other processes at some point in the future. So, just - ** in case this has happened, clear the changeCountDone flag now. - */ - pPager->changeCountDone = pPager->tempFile; +/* +** Return true if the page is already in the journal file. +*/ +static int pageInJournal(PgHdr *pPg){ + return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno); +} +/* +** Read a 32-bit integer from the given file descriptor. Store the integer +** that is read in *pRes. Return SQLITE_OK if everything worked, or an +** error code is something goes wrong. +** +** All values are stored on disk as big-endian. +*/ +static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ + unsigned char ac[4]; + int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset); if( rc==SQLITE_OK ){ - zMaster = pPager->pTmpSpace; - rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); - testcase( rc!=SQLITE_OK ); - } - if( rc==SQLITE_OK && !pPager->noSync - && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) - ){ - rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); - } - if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager, zMaster[0]!='\0'); - testcase( rc!=SQLITE_OK ); - } - if( rc==SQLITE_OK && zMaster[0] && res ){ - /* If there was a master journal and this routine will return success, - ** see if it is possible to delete the master journal. - */ - rc = pager_delmaster(pPager, zMaster); - testcase( rc!=SQLITE_OK ); + *pRes = sqlite3Get4byte(ac); } - - /* The Pager.sectorSize variable may have been updated while rolling - ** back a journal created by a process with a different sector size - ** value. Reset it to the correct value for this process. - */ - setSectorSize(pPager); return rc; } - /* -** Read the content for page pPg out of the database file and into -** pPg->pData. A shared lock or greater must be held on the database -** file before this function is called. -** -** If page 1 is read, then the value of Pager.dbFileVers[] is set to -** the value read from the database file. -** -** If an IO error occurs, then the IO error is returned to the caller. -** Otherwise, SQLITE_OK is returned. +** Write a 32-bit integer into a string buffer in big-endian byte order. */ -static int readDbPage(PgHdr *pPg){ - Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ - Pgno pgno = pPg->pgno; /* Page number to read */ - int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ - int pgsz = pPager->pageSize; /* Number of bytes to read */ +#define put32bits(A,B) sqlite3Put4byte((u8*)A,B) - assert( pPager->eState>=PAGER_READER && !MEMDB ); - assert( isOpen(pPager->fd) ); - if( NEVER(!isOpen(pPager->fd)) ){ - assert( pPager->tempFile ); - memset(pPg->pData, 0, pPager->pageSize); - return SQLITE_OK; - } +/* +** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK +** on success or an error code is something goes wrong. +*/ +static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ + char ac[4]; + put32bits(ac, val); + return sqlite3OsWrite(fd, ac, 4, offset); +} - if( pagerUseWal(pPager) ){ - /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ - i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } - } +/* +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. +*/ +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; - if( pgno==1 ){ - if( rc ){ - /* If the read is unsuccessful, set the dbFileVers[] to something - ** that will never be a valid file version. dbFileVers[] is a copy - ** of bytes 24..39 of the database. Bytes 28..31 should always be - ** zero or the size of the database in page. Bytes 32..35 and 35..39 - ** should be page numbers which are never 0xffffffff. So filling - ** pPager->dbFileVers[] with all 0xff bytes should suffice. - ** - ** For an encrypted database, the situation is more complex: bytes - ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so - ** we should still be ok. - */ - memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); - }else{ - u8 *dbFileVers = &((u8*)pPg->pData)[24]; - memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + assert( !pPager->exclusiveMode || pPager->eLock==eLock ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); - - PAGER_INCR(sqlite3_pager_readdb_count); - PAGER_INCR(pPager->nRead); - IOTRACE(("PGIN %p %d\n", pPager, pgno)); - PAGERTRACE(("FETCH %d page %d hash(%08x)\n", - PAGERID(pPager), pgno, pager_pagehash(pPg))); - return rc; } -#ifndef SQLITE_OMIT_WAL /* -** This function is invoked once for each page that has already been -** written into the log file when a WAL transaction is rolled back. -** Parameter iPg is the page number of said page. The pCtx argument -** is actually a pointer to the Pager structure. +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. ** -** If page iPg is present in the cache, and has no outstanding references, -** it is discarded. Otherwise, if there are one or more outstanding -** references, the page content is reloaded from the database. If the -** attempt to reload content from the database is required and fails, -** return an SQLite error code. Otherwise, SQLITE_OK. +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. */ -static int pagerUndoCallback(void *pCtx, Pgno iPg){ +static int pagerLockDb(Pager *pPager, int eLock){ int rc = SQLITE_OK; - Pager *pPager = (Pager *)pCtx; - PgHdr *pPg; - pPg = sqlite3PagerLookup(pPager, iPg); - if( pPg ){ - if( sqlite3PcachePageRefcount(pPg)==1 ){ - sqlite3PcacheDrop(pPg); - }else{ - rc = readDbPage(pPg); - if( rc==SQLITE_OK ){ - pPager->xReiniter(pPg); - } - sqlite3PagerUnref(pPg); - } + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } } - - /* Normally, if a transaction is rolled back, any backup processes are - ** updated as data is copied out of the rollback journal and into the - ** database. This is not generally possible with a WAL database, as - ** rollback involves simply truncating the log file. Therefore, if one - ** or more frames have already been written to the log (and therefore - ** also copied into the backup databases) as part of this transaction, - ** the backups must be restarted. - */ - sqlite3BackupRestart(pPager->pBackup); - return rc; } /* -** This function is called to rollback a transaction on a WAL database. +** This function determines whether or not the atomic-write optimization +** can be used with this pager. The optimization can be used if: +** +** (a) the value returned by OsDeviceCharacteristics() indicates that +** a database page may be written atomically, and +** (b) the value returned by OsSectorSize() is less than or equal +** to the page size. +** +** The optimization is also always enabled for temporary files. It is +** an error to call this function if pPager is opened on an in-memory +** database. +** +** If the optimization cannot be used, 0 is returned. If it can be used, +** then the value returned is the size of the journal file when it +** contains rollback data for exactly one page. */ -static int pagerRollbackWal(Pager *pPager){ - int rc; /* Return Code */ - PgHdr *pList; /* List of dirty pages to revert */ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +static int jrnlBufferSize(Pager *pPager){ + assert( !MEMDB ); + if( !pPager->tempFile ){ + int dc; /* Device characteristics */ + int nSector; /* Sector size */ + int szPage; /* Page size */ - /* For all pages in the cache that are currently dirty or have already - ** been written (but not committed) to the log file, do one of the - ** following: - ** - ** + Discard the cached page (if refcount==0), or - ** + Reload page content from the database (if refcount>0). - */ - pPager->dbSize = pPager->dbOrigSize; - rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); - pList = sqlite3PcacheDirtyList(pPager->pPCache); - while( pList && rc==SQLITE_OK ){ - PgHdr *pNext = pList->pDirty; - rc = pagerUndoCallback((void *)pPager, pList->pgno); - pList = pNext; + assert( isOpen(pPager->fd) ); + dc = sqlite3OsDeviceCharacteristics(pPager->fd); + nSector = pPager->sectorSize; + szPage = pPager->pageSize; + + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){ + return 0; + } } - return rc; + return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#endif /* -** This function is a wrapper around sqlite3WalFrames(). As well as logging -** the contents of the list of pages headed by pList (connected by pDirty), -** this function notifies any active backup processes that the pages have -** changed. -*/ -static int pagerWalFrames( - Pager *pPager, /* Pager object */ - PgHdr *pList, /* List of frames to log */ - Pgno nTruncate, /* Database size after this commit */ - int isCommit, /* True if this is a commit */ - int syncFlags /* Flags to pass to OsSync() (or 0) */ -){ - int rc; /* Return code */ - - assert( pPager->pWal ); - rc = sqlite3WalFrames(pPager->pWal, - pPager->pageSize, pList, nTruncate, isCommit, syncFlags - ); - if( rc==SQLITE_OK && pPager->pBackup ){ - PgHdr *p; - for(p=pList; p; p=p->pDirty){ - sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); - } - } - +** If SQLITE_CHECK_PAGES is defined then we do some sanity checking +** on the cache using a hash function. This is used for testing +** and debugging only. +*/ #ifdef SQLITE_CHECK_PAGES - { - PgHdr *p; - for(p=pList; p; p=p->pDirty) pager_set_pagehash(p); +/* +** Return a 32-bit hash of the page data for pPage. +*/ +static u32 pager_datahash(int nByte, unsigned char *pData){ + u32 hash = 0; + int i; + for(i=0; ipPager->pageSize, (unsigned char *)pPage->pData); +} +static void pager_set_pagehash(PgHdr *pPage){ + pPage->pageHash = pager_pagehash(pPage); } /* -** Begin a read transaction on the WAL. -** -** This routine used to be called "pagerOpenSnapshot()" because it essentially -** makes a snapshot of the database at the current point in time and preserves -** that snapshot for use by the reader in spite of concurrently changes by -** other writers or checkpointers. +** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES +** is defined, and NDEBUG is not defined, an assert() statement checks +** that the page is either dirty or still matches the calculated page-hash. */ -static int pagerBeginReadTransaction(Pager *pPager){ - int rc; /* Return code */ - int changed = 0; /* True if cache must be reset */ +#define CHECK_PAGE(x) checkPage(x) +static void checkPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); +} - assert( pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); +#else +#define pager_datahash(X,Y) 0 +#define pager_pagehash(X) 0 +#define pager_set_pagehash(X) +#define CHECK_PAGE(x) +#endif /* SQLITE_CHECK_PAGES */ - /* sqlite3WalEndReadTransaction() was not called for the previous - ** transaction in locking_mode=EXCLUSIVE. So call it now. If we - ** are in locking_mode=NORMAL and EndRead() was previously called, - ** the duplicate call is harmless. - */ - sqlite3WalEndReadTransaction(pPager->pWal); +/* +** When this is called the journal file for pager pPager must be open. +** This function attempts to read a master journal file name from the +** end of the file and, if successful, copies it into memory supplied +** by the caller. See comments above writeMasterJournal() for the format +** used to store a master journal file name at the end of a journal file. +** +** zMaster must point to a buffer of at least nMaster bytes allocated by +** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is +** enough space to write the master journal name). If the master journal +** name in the journal is longer than nMaster bytes (including a +** nul-terminator), then this is handled as if no master journal name +** were present in the journal. +** +** If a master journal file name is present at the end of the journal +** file, then it is copied into the buffer pointed to by zMaster. A +** nul-terminator byte is appended to the buffer following the master +** journal file name. +** +** If it is determined that no master journal file name is present +** zMaster[0] is set to 0 and SQLITE_OK returned. +** +** If an error occurs while reading from the journal file, an SQLite +** error code is returned. +*/ +static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ + int rc; /* Return code */ + u32 len; /* Length in bytes of master journal name */ + i64 szJ; /* Total size in bytes of journal file pJrnl */ + u32 cksum; /* MJ checksum value read from journal */ + u32 u; /* Unsigned loop counter */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + zMaster[0] = '\0'; - rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); - if( rc!=SQLITE_OK || changed ){ - pager_reset(pPager); + if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) + || szJ<16 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) + || len>=nMaster + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) + || memcmp(aMagic, aJournalMagic, 8) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) + ){ + return rc; } - return rc; + /* See if the checksum matches the master journal name */ + for(u=0; ujournalOff, assuming a sector +** size of pPager->sectorSize bytes. ** -** If no error occurs, SQLITE_OK is returned and the size of the database -** in pages is stored in *pnPage. Otherwise, an error code (perhaps -** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +** i.e for a sector size of 512: +** +** Pager.journalOff Return value +** --------------------------------------- +** 0 0 +** 512 512 +** 100 512 +** 2000 2048 +** */ -static int pagerPagecount(Pager *pPager, Pgno *pnPage){ - Pgno nPage; /* Value to return via *pnPage */ - - /* Query the WAL sub-system for the database size. The WalDbsize() - ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or - ** if the database size is not available. The database size is not - ** available from the WAL sub-system if the log file is empty or - ** contains no valid committed transactions. - */ - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); - nPage = sqlite3WalDbsize(pPager->pWal); - - /* If the database size was not available from the WAL sub-system, - ** determine it based on the size of the database file. If the size - ** of the database file is not an integer multiple of the page-size, - ** round down to the nearest page. Except, any file larger than 0 - ** bytes in size is considered to contain at least one page. - */ - if( nPage==0 ){ - i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } - } - nPage = (Pgno)(n / pPager->pageSize); - if( nPage==0 && n>0 ){ - nPage = 1; - } - } - - /* If the current number of pages in the file is greater than the - ** configured maximum pager number, increase the allowed limit so - ** that the file can be read. - */ - if( nPage>pPager->mxPgno ){ - pPager->mxPgno = (Pgno)nPage; +static i64 journalHdrOffset(Pager *pPager){ + i64 offset = 0; + i64 c = pPager->journalOff; + if( c ){ + offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager); } - - *pnPage = nPage; - return SQLITE_OK; + assert( offset%JOURNAL_HDR_SZ(pPager)==0 ); + assert( offset>=c ); + assert( (offset-c)eState==PAGER_OPEN ); - assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); - - if( !pPager->tempFile ){ - int isWal; /* True if WAL file exists */ - Pgno nPage; /* Size of the database file */ +static int zeroJournalHdr(Pager *pPager, int doTruncate){ + int rc = SQLITE_OK; /* Return code */ + assert( isOpen(pPager->jfd) ); + if( pPager->journalOff ){ + const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ - rc = pagerPagecount(pPager, &nPage); - if( rc ) return rc; - if( nPage==0 ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); - isWal = 0; + IOTRACE(("JZEROHDR %p\n", pPager)) + if( doTruncate || iLimit==0 ){ + rc = sqlite3OsTruncate(pPager->jfd, 0); }else{ - rc = sqlite3OsAccess( - pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal - ); + static const char zeroHdr[28] = {0}; + rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); } - if( rc==SQLITE_OK ){ - if( isWal ){ - testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); - rc = sqlite3PagerOpenWal(pPager, 0); - }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ - pPager->journalMode = PAGER_JOURNALMODE_DELETE; + if( rc==SQLITE_OK && !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); + } + + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for + ** the persistent journal and the journal file currently consumes more + ** space than that limit allows for, truncate it now. There is no need + ** to sync the file following this operation. + */ + if( rc==SQLITE_OK && iLimit>0 ){ + i64 sz; + rc = sqlite3OsFileSize(pPager->jfd, &sz); + if( rc==SQLITE_OK && sz>iLimit ){ + rc = sqlite3OsTruncate(pPager->jfd, iLimit); } } } return rc; } -#endif /* -** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback -** the entire master journal file. The case pSavepoint==NULL occurs when -** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction -** savepoint. -** -** When pSavepoint is not NULL (meaning a non-transaction savepoint is -** being rolled back), then the rollback consists of up to three stages, -** performed in the order specified: -** -** * Pages are played back from the main journal starting at byte -** offset PagerSavepoint.iOffset and continuing to -** PagerSavepoint.iHdrOffset, or to the end of the main journal -** file if PagerSavepoint.iHdrOffset is zero. -** -** * If PagerSavepoint.iHdrOffset is not zero, then pages are played -** back starting from the journal header immediately following -** PagerSavepoint.iHdrOffset to the end of the main journal file. -** -** * Pages are then played back from the sub-journal file, starting -** with the PagerSavepoint.iSubRec and continuing to the end of -** the journal file. -** -** Throughout the rollback process, each time a page is rolled back, the -** corresponding bit is set in a bitvec structure (variable pDone in the -** implementation below). This is used to ensure that a page is only -** rolled back the first time it is encountered in either journal. -** -** If pSavepoint is NULL, then pages are only played back from the main -** journal file. There is no need for a bitvec in this case. +** The journal file must be open when this routine is called. A journal +** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the +** current location. ** -** In either case, before playback commences the Pager.dbSize variable -** is reset to the value that it held at the start of the savepoint -** (or transaction). No page with a page-number greater than this value -** is played back. If one is encountered it is simply skipped. +** The format for the journal header is as follows: +** - 8 bytes: Magic identifying journal format. +** - 4 bytes: Number of records in journal, or -1 no-sync mode is on. +** - 4 bytes: Random number used for page hash. +** - 4 bytes: Initial database page count. +** - 4 bytes: Sector size used by the process that wrote this journal. +** - 4 bytes: Database page size. +** +** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. */ -static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ - i64 szJ; /* Effective size of the main journal */ - i64 iHdrOff; /* End of first segment of main-journal records */ - int rc = SQLITE_OK; /* Return code */ - Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ +static int writeJournalHdr(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ + u32 nWrite; /* Bytes of header sector written */ + int ii; /* Loop counter */ - assert( pPager->eState!=PAGER_ERROR ); - assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ - /* Allocate a bitvec to use to store the set of pages rolled back */ - if( pSavepoint ){ - pDone = sqlite3BitvecCreate(pSavepoint->nOrig); - if( !pDone ){ - return SQLITE_NOMEM; - } + if( nHeader>JOURNAL_HDR_SZ(pPager) ){ + nHeader = JOURNAL_HDR_SZ(pPager); } - /* Set the database size back to the value it was before the savepoint - ** being reverted was opened. + /* If there are active savepoints and any of them were created + ** since the most recent journal header was written, update the + ** PagerSavepoint.iHdrOffset fields now. */ - pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; - pPager->changeCountDone = pPager->tempFile; - - if( !pSavepoint && pagerUseWal(pPager) ){ - return pagerRollbackWal(pPager); + for(ii=0; iinSavepoint; ii++){ + if( pPager->aSavepoint[ii].iHdrOffset==0 ){ + pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff; + } } - /* Use pPager->journalOff as the effective size of the main rollback - ** journal. The actual file might be larger than this in - ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything - ** past pPager->journalOff is off-limits to us. - */ - szJ = pPager->journalOff; - assert( pagerUseWal(pPager)==0 || szJ==0 ); + pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); - /* Begin by rolling back records from the main journal starting at - ** PagerSavepoint.iOffset and continuing to the next journal header. - ** There might be records in the main journal that have a page number - ** greater than the current database size (pPager->dbSize) but those - ** will be skipped automatically. Pages are added to pDone as they - ** are played back. + /* + ** Write the nRec Field - the number of page records that follow this + ** journal header. Normally, zero is written to this value at this time. + ** After the records are added to the journal (and the journal synced, + ** if in full-sync mode), the zero is overwritten with the true number + ** of records (see syncJournal()). + ** + ** A faster alternative is to write 0xFFFFFFFF to the nRec field. When + ** reading the journal this value tells SQLite to assume that the + ** rest of the journal file contains valid page records. This assumption + ** is dangerous, as if a failure occurred whilst writing to the journal + ** file it may contain some garbage data. There are two scenarios + ** where this risk can be ignored: + ** + ** * When the pager is in no-sync mode. Corruption can follow a + ** power failure in this case anyway. + ** + ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees + ** that garbage data is never appended to the journal file. */ - if( pSavepoint && !pagerUseWal(pPager) ){ - iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; - pPager->journalOff = pSavepoint->iOffset; - while( rc==SQLITE_OK && pPager->journalOffjournalOff, pDone, 1, 1); - } - assert( rc!=SQLITE_DONE ); + assert( isOpen(pPager->fd) || pPager->noSync ); + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); }else{ - pPager->journalOff = 0; + memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* Continue rolling back records out of the main journal starting at - ** the first journal header seen and continuing until the effective end - ** of the main journal file. Continue to skip out-of-range pages and - ** continue adding pages rolled back to pDone. - */ - while( rc==SQLITE_OK && pPager->journalOffcksumInit), &pPager->cksumInit); + put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ + put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); + /* The assumed sector size for this process */ + put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); - /* - ** The "pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff" - ** test is related to ticket #2565. See the discussion in the - ** pager_playback() function for additional information. - */ - if( nJRec==0 - && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff - ){ - nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); - } - for(ii=0; rc==SQLITE_OK && iijournalOffjournalOff, pDone, 1, 1); - } - assert( rc!=SQLITE_DONE ); - } - assert( rc!=SQLITE_OK || pPager->journalOff>=szJ ); + /* The page size */ + put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize); - /* Finally, rollback pages from the sub-journal. Page that were - ** previously rolled back out of the main journal (and are hence in pDone) - ** will be skipped. Out-of-range pages are also skipped. + /* Initializing the tail of the buffer is not necessary. Everything + ** works find if the following memset() is omitted. But initializing + ** the memory prevents valgrind from complaining, so we are willing to + ** take the performance hit. */ - if( pSavepoint ){ - u32 ii; /* Loop counter */ - i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); - - if( pagerUseWal(pPager) ){ - rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); - } - for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ - assert( offset==ii*(4+pPager->pageSize) ); - rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); - } - assert( rc!=SQLITE_DONE ); - } + memset(&zHeader[sizeof(aJournalMagic)+20], 0, + nHeader-(sizeof(aJournalMagic)+20)); - sqlite3BitvecDestroy(pDone); - if( rc==SQLITE_OK ){ - pPager->journalOff = szJ; + /* In theory, it is only necessary to write the 28 bytes that the + ** journal header consumes to the journal file here. Then increment the + ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next + ** record is written to the following sector (leaving a gap in the file + ** that will be implicitly filled in by the OS). + ** + ** However it has been discovered that on some systems this pattern can + ** be significantly slower than contiguously writing data to the file, + ** even if that means explicitly writing data to the block of + ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what + ** is done. + ** + ** The loop is required here in case the sector-size is larger than the + ** database page size. Since the zHeader buffer is only Pager.pageSize + ** bytes in size, more than one call to sqlite3OsWrite() may be required + ** to populate the entire journal header sector. + */ + for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) + rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); + assert( pPager->journalHdr <= pPager->journalOff ); + pPager->journalOff += nHeader; } return rc; } /* -** Change the maximum number of in-memory pages that are allowed. -*/ -SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ - sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); -} - -/* -** Adjust the robustness of the database to damage due to OS crashes -** or power failures by changing the number of syncs()s when writing -** the rollback journal. There are three levels: -** -** OFF sqlite3OsSync() is never called. This is the default -** for temporary and transient files. -** -** NORMAL The journal is synced once before writes begin on the -** database. This is normally adequate protection, but -** it is theoretically possible, though very unlikely, -** that an inopertune power failure could leave the journal -** in a state which would cause damage to the database -** when it is rolled back. -** -** FULL The journal is synced twice before writes begin on the -** database (with some additional information - the nRec field -** of the journal header - being written in between the two -** syncs). If we assume that writing a -** single disk sector is atomic, then this mode provides -** assurance that the journal will not be corrupted to the -** point of causing damage to the database during rollback. -** -** The above is for a rollback-journal mode. For WAL mode, OFF continues -** to mean that no syncs ever occur. NORMAL means that the WAL is synced -** prior to the start of checkpoint and that the database file is synced -** at the conclusion of the checkpoint if the entire content of the WAL -** was written back into the database. But no sync operations occur for -** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL -** file is synced following each commit operation, in addition to the -** syncs associated with NORMAL. +** The journal file must be open when this is called. A journal header file +** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal +** file. The current location in the journal file is given by +** pPager->journalOff. See comments above function writeJournalHdr() for +** a description of the journal header format. ** -** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The -** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync -** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an -** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL -** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the -** synchronous=FULL versus synchronous=NORMAL setting determines when -** the xSync primitive is called and is relevant to all platforms. +** If the header is read successfully, *pNRec is set to the number of +** page records following this header and *pDbSize is set to the size of the +** database before the transaction began, in pages. Also, pPager->cksumInit +** is set to the value read from the journal header. SQLITE_OK is returned +** in this case. ** -** Numeric values associated with these states are OFF==1, NORMAL=2, -** and FULL=3. +** If the journal header file appears to be corrupted, SQLITE_DONE is +** returned and *pNRec and *PDbSize are undefined. If JOURNAL_HDR_SZ bytes +** cannot be read from the journal file an error code is returned. */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel( - Pager *pPager, /* The pager to set safety level for */ - int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ - int bFullFsync, /* PRAGMA fullfsync */ - int bCkptFullFsync /* PRAGMA checkpoint_fullfsync */ +static int readJournalHdr( + Pager *pPager, /* Pager object */ + int isHot, + i64 journalSize, /* Size of the open journal file in bytes */ + u32 *pNRec, /* OUT: Value read from the nRec field */ + u32 *pDbSize /* OUT: Value of original database size field */ ){ - assert( level>=1 && level<=3 ); - pPager->noSync = (level==1 || pPager->tempFile) ?1:0; - pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; - if( pPager->noSync ){ - pPager->syncFlags = 0; - pPager->ckptSyncFlags = 0; - }else if( bFullFsync ){ - pPager->syncFlags = SQLITE_SYNC_FULL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; - }else if( bCkptFullFsync ){ - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; - }else{ - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + int rc; /* Return code */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + i64 iHdrOff; /* Offset of journal header being read */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + /* Advance Pager.journalOff to the start of the next sector. If the + ** journal file is too small for there to be a header stored at this + ** point, return SQLITE_DONE. + */ + pPager->journalOff = journalHdrOffset(pPager); + if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){ + return SQLITE_DONE; } -} -#endif + iHdrOff = pPager->journalOff; -/* -** The following global variable is incremented whenever the library -** attempts to open a temporary file. This information is used for -** testing and analysis only. -*/ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_opentemp_count = 0; -#endif + /* Read in the first 8 bytes of the journal header. If they do not match + ** the magic string found at the start of each journal header, return + ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise, + ** proceed. + */ + if( isHot || iHdrOff!=pPager->journalHdr ){ + rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff); + if( rc ){ + return rc; + } + if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){ + return SQLITE_DONE; + } + } -/* -** Open a temporary file. -** -** Write the file descriptor into *pFile. Return SQLITE_OK on success -** or some other error code if we fail. The OS will automatically -** delete the temporary file when it is closed. -** -** The flags passed to the VFS layer xOpen() call are those specified -** by parameter vfsFlags ORed with the following: -** -** SQLITE_OPEN_READWRITE -** SQLITE_OPEN_CREATE -** SQLITE_OPEN_EXCLUSIVE -** SQLITE_OPEN_DELETEONCLOSE -*/ -static int pagerOpentemp( - Pager *pPager, /* The pager object */ - sqlite3_file *pFile, /* Write the file descriptor here */ - int vfsFlags /* Flags passed through to the VFS */ -){ - int rc; /* Return code */ + /* Read the first three 32-bit fields of the journal header: The nRec + ** field, the checksum-initializer and the database size at the start + ** of the transaction. Return an error code if anything goes wrong. + */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+12, &pPager->cksumInit)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+16, pDbSize)) + ){ + return rc; + } -#ifdef SQLITE_TEST - sqlite3_opentemp_count++; /* Used for testing and analysis only */ -#endif + if( pPager->journalOff==0 ){ + u32 iPageSize; /* Page-size field of journal header */ + u32 iSectorSize; /* Sector-size field of journal header */ - vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; - rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); - assert( rc!=SQLITE_OK || isOpen(pFile) ); + /* Read the page-size and sector-size journal header fields. */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize)) + ){ + return rc; + } + + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } + + /* Check that the values read from the page-size and sector-size fields + ** are within range. To be 'in range', both values need to be a power + ** of two greater than or equal to 512 or 32, and not greater than their + ** respective compile time maximum limits. + */ + if( iPageSize<512 || iSectorSize<32 + || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE + || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 + ){ + /* If the either the page-size or sector-size in the journal-header is + ** invalid, then the process that wrote the journal-header must have + ** crashed before the header was synced. In this case stop reading + ** the journal file here. + */ + return SQLITE_DONE; + } + + /* Update the page-size to match the value read from the journal. + ** Use a testcase() macro to make sure that malloc failure within + ** PagerSetPagesize() is tested. + */ + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); + testcase( rc!=SQLITE_OK ); + + /* Update the assumed sector-size to match the value used by + ** the process that created this journal. If this journal was + ** created by a process other than this one, then this routine + ** is being called from within pager_playback(). The local value + ** of Pager.sectorSize is restored at the end of that routine. + */ + pPager->sectorSize = iSectorSize; + } + + pPager->journalOff += JOURNAL_HDR_SZ(pPager); return rc; } -/* -** Set the busy handler function. -** -** The pager invokes the busy-handler if sqlite3OsLock() returns -** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, -** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE -** lock. It does *not* invoke the busy handler when upgrading from -** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE -** (which occurs during hot-journal rollback). Summary: -** -** Transition | Invokes xBusyHandler -** -------------------------------------------------------- -** NO_LOCK -> SHARED_LOCK | Yes -** SHARED_LOCK -> RESERVED_LOCK | No -** SHARED_LOCK -> EXCLUSIVE_LOCK | No -** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes -** -** If the busy-handler callback returns non-zero, the lock is -** retried. If it returns zero, then the SQLITE_BUSY error is -** returned to the caller of the pager API function. -*/ -SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( - Pager *pPager, /* Pager object */ - int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ - void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ -){ - pPager->xBusyHandler = xBusyHandler; - pPager->pBusyHandlerArg = pBusyHandlerArg; -} /* -** Change the page size used by the Pager object. The new page size -** is passed in *pPageSize. -** -** If the pager is in the error state when this function is called, it -** is a no-op. The value returned is the error state error code (i.e. -** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). -** -** Otherwise, if all of the following are true: -** -** * the new page size (value of *pPageSize) is valid (a power -** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and -** -** * there are no outstanding page references, and -** -** * the database is either not an in-memory database or it is -** an in-memory database that currently consists of zero pages. +** Write the supplied master journal name into the journal file for pager +** pPager at the current location. The master journal name must be the last +** thing written to a journal file. If the pager is in full-sync mode, the +** journal file descriptor is advanced to the next sector boundary before +** anything is written. The format is: ** -** then the pager object page size is set to *pPageSize. +** + 4 bytes: PAGER_MJ_PGNO. +** + N bytes: Master journal filename in utf-8. +** + 4 bytes: N (length of master journal name in bytes, no nul-terminator). +** + 4 bytes: Master journal name checksum. +** + 8 bytes: aJournalMagic[]. ** -** If the page size is changed, then this function uses sqlite3PagerMalloc() -** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt -** fails, SQLITE_NOMEM is returned and the page size remains unchanged. -** In all other cases, SQLITE_OK is returned. +** The master journal page checksum is the sum of the bytes in the master +** journal name, where each byte is interpreted as a signed 8-bit integer. ** -** If the page size is not changed, either because one of the enumerated -** conditions above is not true, the pager was in error state when this -** function was called, or because the memory allocation attempt failed, -** then *pPageSize is set to the old, retained page size before returning. +** If zMaster is a NULL pointer (occurs for a single database transaction), +** this call is a no-op. */ -SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ - int rc = SQLITE_OK; +static int writeMasterJournal(Pager *pPager, const char *zMaster){ + int rc; /* Return code */ + int nMaster; /* Length of string zMaster */ + i64 iHdrOff; /* Offset of header in journal file */ + i64 jrnlSize; /* Size of journal file on disk */ + u32 cksum = 0; /* Checksum of string zMaster */ - /* It is not possible to do a full assert_pager_state() here, as this - ** function may be called from within PagerOpen(), before the state - ** of the Pager object is internally consistent. - ** - ** At one point this function returned an error if the pager was in - ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that - ** there is at least one outstanding page reference, this function - ** is a no-op for that case anyhow. - */ + assert( pPager->setMaster==0 ); + assert( !pagerUseWal(pPager) ); - u32 pageSize = *pPageSize; - assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); - if( (pPager->memDb==0 || pPager->dbSize==0) - && sqlite3PcacheRefCount(pPager->pPCache)==0 - && pageSize && pageSize!=(u32)pPager->pageSize + if( !zMaster + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_OFF ){ - char *pNew = NULL; /* New temp space */ - i64 nByte = 0; + return SQLITE_OK; + } + pPager->setMaster = 1; + assert( isOpen(pPager->jfd) ); + assert( pPager->journalHdr <= pPager->journalOff ); - if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ - rc = sqlite3OsFileSize(pPager->fd, &nByte); - } - if( rc==SQLITE_OK ){ - pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM; - } + /* Calculate the length in bytes and the checksum of zMaster */ + for(nMaster=0; zMaster[nMaster]; nMaster++){ + cksum += zMaster[nMaster]; + } - if( rc==SQLITE_OK ){ - pager_reset(pPager); - pPager->dbSize = (Pgno)(nByte/pageSize); - pPager->pageSize = pageSize; - sqlite3PageFree(pPager->pTmpSpace); - pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); - } + /* If in full-sync mode, advance to the next disk sector before writing + ** the master journal name. This is in case the previous page written to + ** the journal has already been synced. + */ + if( pPager->fullSync ){ + pPager->journalOff = journalHdrOffset(pPager); } + iHdrOff = pPager->journalOff; - *pPageSize = pPager->pageSize; - if( rc==SQLITE_OK ){ - if( nReserve<0 ) nReserve = pPager->nReserve; - assert( nReserve>=0 && nReserve<1000 ); - pPager->nReserve = (i16)nReserve; - pagerReportSize(pPager); + /* Write the master journal data to the end of the journal file. If + ** an error occurs, return the error code to the caller. + */ + if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) + ){ + return rc; + } + pPager->journalOff += (nMaster+20); + + /* If the pager is in peristent-journal mode, then the physical + ** journal-file may extend past the end of the master-journal name + ** and 8 bytes of magic data just written to the file. This is + ** dangerous because the code to rollback a hot-journal file + ** will not be able to find the master-journal name to determine + ** whether or not the journal is hot. + ** + ** Easiest thing to do in this scenario is to truncate the journal + ** file to the required size. + */ + if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) + && jrnlSize>pPager->journalOff + ){ + rc = sqlite3OsTruncate(pPager->jfd, pPager->journalOff); } return rc; } /* -** Return a pointer to the "temporary page" buffer held internally -** by the pager. This is a buffer that is big enough to hold the -** entire content of a database page. This buffer is used internally -** during rollback and will be overwritten whenever a rollback -** occurs. But other modules are free to use it too, as long as -** no rollbacks are happening. +** Find a page in the hash table given its page number. Return +** a pointer to the page or NULL if the requested page is not +** already in memory. */ -SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager *pPager){ - return pPager->pTmpSpace; +static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ + PgHdr *p; /* Return value */ + + /* It is not possible for a call to PcacheFetch() with createFlag==0 to + ** fail, since no attempt to allocate dynamic memory will be made. + */ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); + return p; } /* -** Attempt to set the maximum database page count if mxPage is positive. -** Make no changes if mxPage is zero or negative. And never reduce the -** maximum page count below the current size of the database. -** -** Regardless of mxPage, return the current maximum page count. +** Discard the entire contents of the in-memory page-cache. */ -SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ - if( mxPage>0 ){ - pPager->mxPgno = mxPage; - } - assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ - assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ - return pPager->mxPgno; +static void pager_reset(Pager *pPager){ + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); } /* -** The following set of routines are used to disable the simulated -** I/O error mechanism. These routines are used to avoid simulated -** errors in places where we do not care about errors. -** -** Unless -DSQLITE_TEST=1 is used, these routines are all no-ops -** and generate no code. +** Free all structures in the Pager.aSavepoint[] array and set both +** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal +** if it is open and the pager is not in exclusive mode. */ -#ifdef SQLITE_TEST -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_hit; -static int saved_cnt; -void disable_simulated_io_errors(void){ - saved_cnt = sqlite3_io_error_pending; - sqlite3_io_error_pending = -1; -} -void enable_simulated_io_errors(void){ - sqlite3_io_error_pending = saved_cnt; +static void releaseAllSavepoints(Pager *pPager){ + int ii; /* Iterator for looping through Pager.aSavepoint */ + for(ii=0; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ + sqlite3OsClose(pPager->sjfd); + } + sqlite3_free(pPager->aSavepoint); + pPager->aSavepoint = 0; + pPager->nSavepoint = 0; + pPager->nSubRec = 0; } -#else -# define disable_simulated_io_errors() -# define enable_simulated_io_errors() -#endif /* -** Read the first N bytes from the beginning of the file into memory -** that pDest points to. -** -** If the pager was opened on a transient file (zFilename==""), or -** opened on a file less than N bytes in size, the output buffer is -** zeroed and SQLITE_OK returned. The rationale for this is that this -** function is used to read database headers, and a new transient or -** zero sized database has a header than consists entirely of zeroes. -** -** If any IO error apart from SQLITE_IOERR_SHORT_READ is encountered, -** the error code is returned to the caller and the contents of the -** output buffer undefined. +** Set the bit number pgno in the PagerSavepoint.pInSavepoint +** bitvecs of all open savepoints. Return SQLITE_OK if successful +** or SQLITE_NOMEM if a malloc failure occurs. */ -SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ - int rc = SQLITE_OK; - memset(pDest, 0, N); - assert( isOpen(pPager->fd) || pPager->tempFile ); - - /* This routine is only called by btree immediately after creating - ** the Pager object. There has not been an opportunity to transition - ** to WAL mode yet. - */ - assert( !pagerUseWal(pPager) ); +static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ + int ii; /* Loop counter */ + int rc = SQLITE_OK; /* Result code */ - if( isOpen(pPager->fd) ){ - IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) - rc = sqlite3OsRead(pPager->fd, pDest, N, 0); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; + for(ii=0; iinSavepoint; ii++){ + PagerSavepoint *p = &pPager->aSavepoint[ii]; + if( pgno<=p->nOrig ){ + rc |= sqlite3BitvecSet(p->pInSavepoint, pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); } } return rc; } /* -** This function may only be called when a read-transaction is open on -** the pager. It returns the total number of pages in the database. +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. ** -** However, if the file is between 1 and bytes in size, then -** this is considered a 1 page file. +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). */ -SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ - assert( pPager->eState>=PAGER_READER ); - assert( pPager->eState!=PAGER_WRITER_FINISHED ); - *pnPage = (int)pPager->dbSize; -} +static void pager_unlock(Pager *pPager){ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ + int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; + + /* If the operating system support deletion of open files, then + ** close the journal file when dropping the database lock. Otherwise + ** another connection with journal_mode=delete might delete the file + ** out from under us. + */ + assert( (PAGER_JOURNALMODE_MEMORY & 5)!=1 ); + assert( (PAGER_JOURNALMODE_OFF & 5)!=1 ); + assert( (PAGER_JOURNALMODE_WAL & 5)!=1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) + || 1!=(pPager->journalMode & 5) + ){ + sqlite3OsClose(pPager->jfd); + } + + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. + */ + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; + } + + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. + */ + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + if( pPager->errCode ){ + assert( !MEMDB ); + pager_reset(pPager); + pPager->changeCountDone = pPager->tempFile; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setMaster = 0; +} /* -** Try to obtain a lock of type locktype on the database file. If -** a similar or greater lock is already held, this function is a no-op -** (returning SQLITE_OK immediately). +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. ** -** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke -** the busy callback if the lock is currently not available. Repeat -** until the busy callback returns false or until the attempt to -** obtain the lock succeeds. +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. ** -** Return SQLITE_OK on success and an error code if we cannot obtain -** the lock. If the lock is obtained successfully, set the Pager.state -** variable to locktype before returning. +** The ERROR state indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding +** the contents of the pager-cache. If a transaction was active when +** the persistent error occurred, then the rollback journal may need +** to be replayed to restore the contents of the database file (as if +** it were a hot-journal). */ -static int pager_wait_on_lock(Pager *pPager, int locktype){ - int rc; /* Return code */ - - /* Check that this is either a no-op (because the requested lock is - ** already held, or one of the transistions that the busy-handler - ** may be invoked during, according to the comment above - ** sqlite3PagerSetBusyhandler(). - */ - assert( (pPager->eLock>=locktype) - || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) - || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) +static int pager_error(Pager *pPager, int rc){ + int rc2 = rc & 0xff; + assert( rc==SQLITE_OK || !MEMDB ); + assert( + pPager->errCode==SQLITE_FULL || + pPager->errCode==SQLITE_OK || + (pPager->errCode & 0xff)==SQLITE_IOERR ); - - do { - rc = pagerLockDb(pPager, locktype); - }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ + pPager->errCode = rc; + pPager->eState = PAGER_ERROR; + } return rc; } /* -** Function assertTruncateConstraint(pPager) checks that one of the -** following is true for all dirty pages currently in the page-cache: +** This routine ends a transaction. A transaction is usually ended by +** either a COMMIT or a ROLLBACK operation. This routine may be called +** after rollback of a hot-journal, or if an error occurs while opening +** the journal file or writing the very first journal-header of a +** database transaction. +** +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. ** -** a) The page number is less than or equal to the size of the -** current database image, in pages, OR +** Otherwise, any active savepoints are released. ** -** b) if the page content were written at this time, it would not -** be necessary to write the current content out to the sub-journal -** (as determined by function subjRequiresPage()). +** If the journal file is open, then it is "finalized". Once a journal +** file has been finalized it is not possible to use it to roll back a +** transaction. Nor will it be considered to be a hot-journal by this +** or any other database connection. Exactly how a journal is finalized +** depends on whether or not the pager is running in exclusive mode and +** the current journal-mode (Pager.journalMode value), as follows: ** -** If the condition asserted by this function were not true, and the -** dirty page were to be discarded from the cache via the pagerStress() -** routine, pagerStress() would not write the current page content to -** the database file. If a savepoint transaction were rolled back after -** this happened, the correct behaviour would be to restore the current -** content of the page. However, since this content is not present in either -** the database file or the portion of the rollback journal and -** sub-journal rolled back the content could not be restored and the -** database image would become corrupt. It is therefore fortunate that -** this circumstance cannot arise. +** journalMode==MEMORY +** Journal file descriptor is simply closed. This destroys an +** in-memory journal. +** +** journalMode==TRUNCATE +** Journal file is truncated to zero bytes in size. +** +** journalMode==PERSIST +** The first 28 bytes of the journal file are zeroed. This invalidates +** the first journal header in the file, and hence the entire journal +** file. An invalid journal file cannot be rolled back. +** +** journalMode==DELETE +** The journal file is closed and deleted using sqlite3OsDelete(). +** +** If the pager is running in exclusive mode, this method of finalizing +** the journal file is never used. Instead, if the journalMode is +** DELETE and the pager is in exclusive mode, the method described under +** journalMode==PERSIST is used instead. +** +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. +** +** SQLITE_OK is returned if no error occurs. If an error occurs during +** any of the IO operations to finalize the journal file or unlock the +** database then the IO error code is returned to the user. If the +** operation to finalize the journal file fails, then the code still +** tries to unlock the database file if not in exclusive mode. If the +** unlock operation fails as well, then the first error code related +** to the first error encountered (the journal finalization one) is +** returned. */ -#if defined(SQLITE_DEBUG) -static void assertTruncateConstraintCb(PgHdr *pPg){ - assert( pPg->flags&PGHDR_DIRTY ); - assert( !subjRequiresPage(pPg) || pPg->pgno<=pPg->pPager->dbSize ); -} -static void assertTruncateConstraint(Pager *pPager){ - sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); -} -#else -# define assertTruncateConstraint(pPager) -#endif +static int pager_end_transaction(Pager *pPager, int hasMaster){ + int rc = SQLITE_OK; /* Error code from journal finalization operation */ + int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ -/* -** Truncate the in-memory database file image to nPage pages. This -** function does not actually modify the database file on disk. It -** just sets the internal state of the pager object so that the -** truncation will be done when the current transaction is committed. -*/ -SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ - assert( pPager->dbSize>=nPage ); - assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); - pPager->dbSize = nPage; - assertTruncateConstraint(pPager); -} + /* Do nothing if the pager does not have an open write transaction + ** or at least a RESERVED lock. This function may be called when there + ** is no write-transaction active but a RESERVED or greater lock is + ** held under two circumstances: + ** + ** 1. After a successful hot-journal rollback, it is called with + ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. + ** + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** lock switches back to locking_mode=normal and then executes a + ** read-transaction, this function is called with eState==PAGER_READER + ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. + */ + assert( assert_pager_state(pPager) ); + assert( pPager->eState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); + if( isOpen(pPager->jfd) ){ + assert( !pagerUseWal(pPager) ); -/* -** This function is called before attempting a hot-journal rollback. It -** syncs the journal file to disk, then sets pPager->journalHdr to the -** size of the journal file so that the pager_playback() routine knows -** that the entire journal file has been synced. -** -** Syncing a hot-journal to disk before attempting to roll it back ensures -** that if a power-failure occurs during the rollback, the process that -** attempts rollback following system recovery sees the same journal -** content as this process. -** -** If everything goes as planned, SQLITE_OK is returned. Otherwise, -** an SQLite error code. -*/ -static int pagerSyncHotJournal(Pager *pPager){ - int rc = SQLITE_OK; - if( !pPager->noSync ){ - rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_NORMAL); + /* Finalize the journal file. */ + if( sqlite3IsMemJournal(pPager->jfd) ){ + assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + sqlite3OsClose(pPager->jfd); + }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ + if( pPager->journalOff==0 ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsTruncate(pPager->jfd, 0); + } + pPager->journalOff = 0; + }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST + || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + ){ + rc = zeroJournalHdr(pPager, hasMaster); + pPager->journalOff = 0; + }else{ + /* This branch may be executed with Pager.journalMode==MEMORY if + ** a hot-journal was just rolled back. In this case the journal + ** file should be closed and deleted. If this connection writes to + ** the database file, it will do so using an in-memory journal. + */ + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + sqlite3OsClose(pPager->jfd); + if( !pPager->tempFile ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + } } - if( rc==SQLITE_OK ){ - rc = sqlite3OsFileSize(pPager->jfd, &pPager->journalHdr); + +#ifdef SQLITE_CHECK_PAGES + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = pager_lookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnref(p); + } } - return rc; +#endif + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + pPager->nRec = 0; + sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + + if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ + rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); + assert( rc2==SQLITE_OK ); + } + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); + pPager->changeCountDone = 0; + } + pPager->eState = PAGER_READER; + pPager->setMaster = 0; + + return (rc==SQLITE_OK?rc2:rc); } /* -** Shutdown the page cache. Free all memory and close all files. +** Execute a rollback if a transaction is active and unlock the +** database file. ** -** If a transaction was in progress when this routine is called, that -** transaction is rolled back. All outstanding pages are invalidated -** and their memory is freed. Any attempt to use a page associated -** with this page cache after this function returns will likely -** result in a coredump. +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. ** -** This function always succeeds. If a transaction is active an attempt -** is made to roll it back. If an error occurs during the rollback -** a hot journal may be left in the filesystem but no error is returned -** to the caller. +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. */ -SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ - u8 *pTmp = (u8 *)pPager->pTmpSpace; - - disable_simulated_io_errors(); - sqlite3BeginBenignMalloc(); - /* pPager->errCode = 0; */ - pPager->exclusiveMode = 0; -#ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); - pPager->pWal = 0; -#endif - pager_reset(pPager); - if( MEMDB ){ - pager_unlock(pPager); - }else{ - /* If it is open, sync the journal file before calling UnlockAndRollback. - ** If this is not done, then an unsynced portion of the open journal - ** file may be played back into the database. If a power failure occurs - ** while this is happening, the database could become corrupt. - ** - ** If an error occurs while trying to sync the journal, shift the pager - ** into the ERROR state. This causes UnlockAndRollback to unlock the - ** database and close the journal file without attempting to roll it - ** back or finalize it. The next database user will have to do hot-journal - ** rollback before accessing the database file. - */ - if( isOpen(pPager->jfd) ){ - pager_error(pPager, pagerSyncHotJournal(pPager)); +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0); } - pagerUnlockAndRollback(pPager); } - sqlite3EndBenignMalloc(); - enable_simulated_io_errors(); - PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); - IOTRACE(("CLOSE %p\n", pPager)) - sqlite3OsClose(pPager->jfd); - sqlite3OsClose(pPager->fd); - sqlite3PageFree(pTmp); - sqlite3PcacheClose(pPager->pPCache); - -#ifdef SQLITE_HAS_CODEC - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); -#endif - - assert( !pPager->aSavepoint && !pPager->pInJournal ); - assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); - - sqlite3_free(pPager); - return SQLITE_OK; + pager_unlock(pPager); } -#if !defined(NDEBUG) || defined(SQLITE_TEST) /* -** Return the page number for page pPg. +** Parameter aData must point to a buffer of pPager->pageSize bytes +** of data. Compute and return a checksum based ont the contents of the +** page of data and the current value of pPager->cksumInit. +** +** This is not a real checksum. It is really just the sum of the +** random initial value (pPager->cksumInit) and every 200th byte +** of the page data, starting with byte offset (pPager->pageSize%200). +** Each byte is interpreted as an 8-bit unsigned integer. +** +** Changing the formula used to compute this checksum results in an +** incompatible journal file format. +** +** If journal corruption occurs due to a power failure, the most likely +** scenario is that one end or the other of the record will be changed. +** It is much less likely that the two ends of the journal record will be +** correct and the middle be corrupt. Thus, this "checksum" scheme, +** though fast and simple, catches the mostly likely kind of corruption. */ -SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage *pPg){ - return pPg->pgno; +static u32 pager_cksum(Pager *pPager, const u8 *aData){ + u32 cksum = pPager->cksumInit; /* Checksum value to return */ + int i = pPager->pageSize-200; /* Loop counter */ + while( i>0 ){ + cksum += aData[i]; + i -= 200; + } + return cksum; } -#endif /* -** Increment the reference count for page pPg. +** Report the current page size and number of reserved bytes back +** to the codec. */ -SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ - sqlite3PcacheRef(pPg); +#ifdef SQLITE_HAS_CODEC +static void pagerReportSize(Pager *pPager){ + if( pPager->xCodecSizeChng ){ + pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, + (int)pPager->nReserve); + } } +#else +# define pagerReportSize(X) /* No-op if we do not support a codec */ +#endif /* -** Sync the journal. In other words, make sure all the pages that have -** been written to the journal have actually reached the surface of the -** disk and can be restored in the event of a hot-journal rollback. +** Read a single page from either the journal file (if isMainJrnl==1) or +** from the sub-journal (if isMainJrnl==0) and playback that page. +** The page begins at offset *pOffset into the file. The *pOffset +** value is increased to the start of the next page in the journal. ** -** If the Pager.noSync flag is set, then this function is a no-op. -** Otherwise, the actions required depend on the journal-mode and the -** device characteristics of the the file-system, as follows: -** -** * If the journal file is an in-memory journal file, no action need -** be taken. +** The main rollback journal uses checksums - the statement journal does +** not. ** -** * Otherwise, if the device does not support the SAFE_APPEND property, -** then the nRec field of the most recently written journal header -** is updated to contain the number of journal records that have -** been written following it. If the pager is operating in full-sync -** mode, then the journal file is synced before this field is updated. +** If the page number of the page record read from the (sub-)journal file +** is greater than the current value of Pager.dbSize, then playback is +** skipped and SQLITE_OK is returned. ** -** * If the device does not support the SEQUENTIAL property, then -** journal file is synced. +** If pDone is not NULL, then it is a record of pages that have already +** been played back. If the page at *pOffset has already been played back +** (if the corresponding pDone bit is set) then skip the playback. +** Make sure the pDone bit corresponding to the *pOffset page is set +** prior to returning. ** -** Or, in pseudo-code: +** If the page record is successfully read from the (sub-)journal file +** and played back, then SQLITE_OK is returned. If an IO error occurs +** while reading the record from the (sub-)journal file or while writing +** to the database file, then the IO error code is returned. If data +** is successfully read from the (sub-)journal file but appears to be +** corrupted, SQLITE_DONE is returned. Data is considered corrupted in +** two circumstances: +** +** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or +** * If the record is being rolled back from the main journal file +** and the checksum field does not match the record content. ** -** if( NOT ){ -** if( NOT SAFE_APPEND ){ -** if( ) xSync(); -** -** } -** if( NOT SEQUENTIAL ) xSync(); -** } +** Neither of these two scenarios are possible during a savepoint rollback. ** -** If successful, this routine clears the PGHDR_NEED_SYNC flag of every -** page currently held in memory before returning SQLITE_OK. If an IO -** error is encountered, then the IO error code is returned to the caller. +** If this is a savepoint rollback, then memory may have to be dynamically +** allocated by this function. If this is the case and an allocation fails, +** SQLITE_NOMEM is returned. */ -static int syncJournal(Pager *pPager, int newHdr){ - int rc; /* Return code */ - - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - ); - assert( assert_pager_state(pPager) ); - assert( !pagerUseWal(pPager) ); - - rc = sqlite3PagerExclusiveLock(pPager); - if( rc!=SQLITE_OK ) return rc; - - if( !pPager->noSync ){ - assert( !pPager->tempFile ); - if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ - const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); - assert( isOpen(pPager->jfd) ); +static int pager_playback_one_page( + Pager *pPager, /* The pager being played back */ + i64 *pOffset, /* Offset of record to playback */ + Bitvec *pDone, /* Bitvec of pages already played back */ + int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */ + int isSavepnt /* True for a savepoint rollback */ +){ + int rc; + PgHdr *pPg; /* An existing page in the cache */ + Pgno pgno; /* The page number of a page in journal */ + u32 cksum; /* Checksum used for sanity checking */ + char *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ + int isSynced; /* True if journal page is synced */ - if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ - /* This block deals with an obscure problem. If the last connection - ** that wrote to this database was operating in persistent-journal - ** mode, then the journal file may at this point actually be larger - ** than Pager.journalOff bytes. If the next thing in the journal - ** file happens to be a journal-header (written as part of the - ** previous connection's transaction), and a crash or power-failure - ** occurs after nRec is updated but before this connection writes - ** anything else to the journal file (or commits/rolls back its - ** transaction), then SQLite may become confused when doing the - ** hot-journal rollback following recovery. It may roll back all - ** of this connections data, then proceed to rolling back the old, - ** out-of-date data that follows it. Database corruption. - ** - ** To work around this, if the journal file does appear to contain - ** a valid header following Pager.journalOff, then write a 0x00 - ** byte to the start of it to prevent it from being recognized. - ** - ** Variable iNextHdrOffset is set to the offset at which this - ** problematic header will occur, if it exists. aMagic is used - ** as a temporary buffer to inspect the first couple of bytes of - ** the potential journal header. - */ - i64 iNextHdrOffset; - u8 aMagic[8]; - u8 zHeader[sizeof(aJournalMagic)+4]; + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ + assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ + assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ - memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); - put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); + aData = pPager->pTmpSpace; + assert( aData ); /* Temp storage must have already been allocated */ + assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); - iNextHdrOffset = journalHdrOffset(pPager); - rc = sqlite3OsRead(pPager->jfd, aMagic, 8, iNextHdrOffset); - if( rc==SQLITE_OK && 0==memcmp(aMagic, aJournalMagic, 8) ){ - static const u8 zerobyte = 0; - rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, iNextHdrOffset); - } - if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ - return rc; - } + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); - /* Write the nRec value into the journal file header. If in - ** full-synchronous mode, sync the journal first. This ensures that - ** all data has really hit the disk before nRec is updated to mark - ** it as a candidate for rollback. - ** - ** This is not required if the persistent media supports the - ** SAFE_APPEND property. Because in this case it is not possible - ** for garbage data to be appended to the file, the nRec field - ** is populated with 0xFFFFFFFF when the journal header is written - ** and never needs to be updated. - */ - if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ - PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); - IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); - if( rc!=SQLITE_OK ) return rc; - } - IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); - rc = sqlite3OsWrite( - pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr - ); - if( rc!=SQLITE_OK ) return rc; - } - if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ - PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); - IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| - (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) - ); - if( rc!=SQLITE_OK ) return rc; - } + /* Read the page number and page data from the journal or sub-journal + ** file. Return an error code to the caller if an IO error occurs. + */ + jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; + rc = read32bits(jfd, *pOffset, &pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4); + if( rc!=SQLITE_OK ) return rc; + *pOffset += pPager->pageSize + 4 + isMainJrnl*4; - pPager->journalHdr = pPager->journalOff; - if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ - pPager->nRec = 0; - rc = writeJournalHdr(pPager); - if( rc!=SQLITE_OK ) return rc; - } - }else{ - pPager->journalHdr = pPager->journalOff; + /* Sanity checking on the page. This is more important that I originally + ** thought. If a power failure occurs while the journal is being written, + ** it could cause invalid data to be written into the journal. We need to + ** detect this invalid data (with high probability) and ignore it. + */ + if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ + assert( !isSavepnt ); + return SQLITE_DONE; + } + if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){ + return SQLITE_OK; + } + if( isMainJrnl ){ + rc = read32bits(jfd, (*pOffset)-4, &cksum); + if( rc ) return rc; + if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){ + return SQLITE_DONE; } } - /* Unless the pager is in noSync mode, the journal file was just - ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on - ** all pages. + /* If this page has already been played by before during the current + ** rollback, then don't bother to play it back again. */ - sqlite3PcacheClearSyncFlags(pPager->pPCache); - pPager->eState = PAGER_WRITER_DBMOD; - assert( assert_pager_state(pPager) ); - return SQLITE_OK; -} - -/* -** The argument is the first in a linked list of dirty pages connected -** by the PgHdr.pDirty pointer. This function writes each one of the -** in-memory pages in the list to the database file. The argument may -** be NULL, representing an empty list. In this case this function is -** a no-op. -** -** The pager must hold at least a RESERVED lock when this function -** is called. Before writing anything to the database file, this lock -** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, -** SQLITE_BUSY is returned and no data is written to the database file. -** -** If the pager is a temp-file pager and the actual file-system file -** is not yet open, it is created and opened before any data is -** written out. -** -** Once the lock has been upgraded and, if necessary, the file opened, -** the pages are written out to the database file in list order. Writing -** a page is skipped if it meets either of the following criteria: -** -** * The page number is greater than Pager.dbSize, or -** * The PGHDR_DONT_WRITE flag is set on the page. -** -** If writing out a page causes the database file to grow, Pager.dbFileSize -** is updated accordingly. If page 1 is written out, then the value cached -** in Pager.dbFileVers[] is updated to match the new value stored in -** the database file. -** -** If everything is successful, SQLITE_OK is returned. If an IO error -** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot -** be obtained, SQLITE_BUSY is returned. -*/ -static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ - int rc = SQLITE_OK; /* Return code */ - - /* This function is only called for rollback pagers in WRITER_DBMOD state. */ - assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); - assert( pPager->eLock==EXCLUSIVE_LOCK ); + if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ + return rc; + } - /* If the file is a temp-file has not yet been opened, open it now. It - ** is not possible for rc to be other than SQLITE_OK if this branch - ** is taken, as pager_wait_on_lock() is a no-op for temp-files. + /* When playing back page 1, restore the nReserve setting */ - if( !isOpen(pPager->fd) ){ - assert( pPager->tempFile && rc==SQLITE_OK ); - rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); + if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ + pPager->nReserve = ((u8*)aData)[20]; + pagerReportSize(pPager); } - /* Before the first write, give the VFS a hint of what the final - ** file size will be. + /* If the pager is in CACHEMOD state, then there must be a copy of this + ** page in the pager cache. In this case just update the pager cache, + ** not the database file. The page is left marked dirty in this case. + ** + ** An exception to the above rule: If the database is in no-sync mode + ** and a page is moved during an incremental vacuum then the page may + ** not be in the pager cache. Later: if a malloc() or IO error occurs + ** during a Movepage() call, then the page may not be in the cache + ** either. So the condition described in the above paragraph is not + ** assert()able. + ** + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. + ** + ** Ticket #1171: The statement journal might contain page content that is + ** different from the page content at the start of the transaction. + ** This occurs when a page is changed prior to the start of a statement + ** then changed again within the statement. When rolling back such a + ** statement we must not write to the original database unless we know + ** for certain that original page contents are synced into the main rollback + ** journal. Otherwise, a power loss might leave modified data in the + ** database file without an entry in the rollback journal that can + ** restore the database to its original form. Two conditions must be + ** met before writing to the database files. (1) the database must be + ** locked. (2) we know that the original page content is fully synced + ** in the main journal either because the page is not in cache or else + ** the page is marked as needSync==0. + ** + ** 2008-04-14: When attempting to vacuum a corrupt database file, it + ** is possible to fail a statement on a database that does not yet exist. + ** Do not attempt to write if database file has never been opened. */ - assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ - sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); - pPager->dbHintSize = pPager->dbSize; + if( pagerUseWal(pPager) ){ + pPg = 0; + }else{ + pPg = pager_lookup(pPager, pgno); } - - while( rc==SQLITE_OK && pList ){ - Pgno pgno = pList->pgno; - - /* If there are dirty pages in the page cache with page numbers greater - ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to - ** make the file smaller (presumably by auto-vacuum code). Do not write - ** any such pages to the file. + assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", + PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), + (isMainJrnl?"main-journal":"sub-journal") + )); + if( isMainJrnl ){ + isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr); + }else{ + isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); + } + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + && isSynced + ){ + i64 ofst = (pgno-1)*(i64)pPager->pageSize; + testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); + assert( !pagerUseWal(pPager) ); + rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + if( pPager->pBackup ){ + CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); + } + }else if( !isMainJrnl && pPg==0 ){ + /* If this is a rollback of a savepoint and data was not written to + ** the database and the page is not in-memory, there is a potential + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. ** - ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag - ** set (set by sqlite3PagerDontWrite()). + ** There are a couple of different ways this can happen. All are quite + ** obscure. When running in synchronous mode, this can only happen + ** if the page is on the free-list at the start of the transaction, then + ** populated, then moved using sqlite3PagerMovepage(). + ** + ** The solution is to add an in-memory page to the cache containing + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as + ** requiring a journal-sync before it is written. */ - if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ - i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ - char *pData; /* Data to write */ - - assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); - - /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); - - /* Write out the page data. */ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); - - /* If page 1 was just written, update Pager.dbFileVers to match - ** the value now stored in the database file. If writing this - ** page caused the database file to grow, update dbFileSize. + assert( isSavepnt ); + assert( pPager->doNotSpill==0 ); + pPager->doNotSpill++; + rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1); + assert( pPager->doNotSpill==1 ); + pPager->doNotSpill--; + if( rc!=SQLITE_OK ) return rc; + pPg->flags &= ~PGHDR_NEED_READ; + sqlite3PcacheMakeDirty(pPg); + } + if( pPg ){ + /* No page should ever be explicitly rolled back that is in use, except + ** for page 1 which is held in use in order to keep the lock on the + ** database active. However such a page may be rolled back as a result + ** of an internal error resulting in an automatic call to + ** sqlite3PagerRollback(). + */ + void *pData; + pData = pPg->pData; + memcpy(pData, (u8*)aData, pPager->pageSize); + pPager->xReiniter(pPg); + if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ + /* If the contents of this page were just restored from the main + ** journal file, then its content must be as they were when the + ** transaction was first opened. In this case we can mark the page + ** as clean, since there will be no need to write it out to the + ** database. + ** + ** There is one exception to this rule. If the page is being rolled + ** back as part of a savepoint (or statement) rollback from an + ** unsynced portion of the main journal file, then it is not safe + ** to mark the page as clean. This is because marking the page as + ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is + ** already in the journal file (recorded in Pager.pInJournal) and + ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to + ** again within this transaction, it will be marked as dirty but + ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially + ** be written out into the database file before its journal file + ** segment is synced. If a crash occurs during or following this, + ** database corruption may ensue. */ - if( pgno==1 ){ - memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); - } - if( pgno>pPager->dbFileSize ){ - pPager->dbFileSize = pgno; - } - - /* Update any backup objects copying the contents of this pager. */ - sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); - - PAGERTRACE(("STORE %d page %d hash(%08x)\n", - PAGERID(pPager), pgno, pager_pagehash(pList))); - IOTRACE(("PGOUT %p %d\n", pPager, pgno)); - PAGER_INCR(sqlite3_pager_writedb_count); - PAGER_INCR(pPager->nWrite); - }else{ - PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); + assert( !pagerUseWal(pPager) ); + sqlite3PcacheMakeClean(pPg); } - pager_set_pagehash(pList); - pList = pList->pDirty; - } - - return rc; -} + pager_set_pagehash(pPg); -/* -** Ensure that the sub-journal file is open. If it is already open, this -** function is a no-op. -** -** SQLITE_OK is returned if everything goes according to plan. An -** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() -** fails. -*/ -static int openSubJournal(Pager *pPager){ - int rc = SQLITE_OK; - if( !isOpen(pPager->sjfd) ){ - if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ - sqlite3MemJournalOpen(pPager->sjfd); - }else{ - rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + /* If this was page 1, then restore the value of Pager.dbFileVers. + ** Do this before any decoding. */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); } + + /* Decode the page just read from disk */ + CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); + sqlite3PcacheRelease(pPg); } return rc; } /* -** Append a record of the current state of page pPg to the sub-journal. -** It is the callers responsibility to use subjRequiresPage() to check -** that it is really required before calling this function. +** Parameter zMaster is the name of a master journal file. A single journal +** file that referred to the master journal file has just been rolled back. +** This routine checks if it is possible to delete the master journal file, +** and does so if it is. ** -** If successful, set the bit corresponding to pPg->pgno in the bitvecs -** for all open savepoints before returning. +** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not +** available for use within this function. ** -** This function returns SQLITE_OK if everything is successful, an IO -** error code if the attempt to write to the sub-journal fails, or -** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint -** bitvec. +** When a master journal file is created, it is populated with the names +** of all of its child journals, one after another, formatted as utf-8 +** encoded text. The end of each child journal file is marked with a +** nul-terminator byte (0x00). i.e. the entire contents of a master journal +** file for a transaction involving two databases might be: +** +** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" +** +** A master journal file may only be deleted once all of its child +** journals have been rolled back. +** +** This function reads the contents of the master-journal file into +** memory and loops through each of the child journal names. For +** each child journal, it checks if: +** +** * if the child journal exists, and if so +** * if the child journal contains a reference to master journal +** file zMaster +** +** If a child journal can be found that matches both of the criteria +** above, this function returns without doing anything. Otherwise, if +** no such child journal can be found, file zMaster is deleted from +** the file-system using sqlite3OsDelete(). +** +** If an IO error within this function, an error code is returned. This +** function allocates memory by calling sqlite3Malloc(). If an allocation +** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors +** occur, SQLITE_OK is returned. +** +** TODO: This function allocates a single block of memory to load +** the entire contents of the master journal file. This could be +** a couple of kilobytes or so - potentially larger than the page +** size. */ -static int subjournalPage(PgHdr *pPg){ - int rc = SQLITE_OK; - Pager *pPager = pPg->pPager; - if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ +static int pager_delmaster(Pager *pPager, const char *zMaster){ + sqlite3_vfs *pVfs = pPager->pVfs; + int rc; /* Return code */ + sqlite3_file *pMaster; /* Malloc'd master-journal file descriptor */ + sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */ + char *zMasterJournal = 0; /* Contents of master journal file */ + i64 nMasterJournal; /* Size of master journal file */ + char *zJournal; /* Pointer to one journal within MJ file */ + char *zMasterPtr; /* Space to hold MJ filename from a journal file */ + int nMasterPtr; /* Amount of space allocated to zMasterPtr[] */ - /* Open the sub-journal, if it has not already been opened */ - assert( pPager->useJournal ); - assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); - assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); - assert( pagerUseWal(pPager) - || pageInJournal(pPg) - || pPg->pgno>pPager->dbOrigSize - ); - rc = openSubJournal(pPager); + /* Allocate space for both the pJournal and pMaster file descriptors. + ** If successful, open the master journal file for reading. + */ + pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); + if( !pMaster ){ + rc = SQLITE_NOMEM; + }else{ + const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); + } + if( rc!=SQLITE_OK ) goto delmaster_out; - /* If the sub-journal was opened successfully (or was already open), - ** write the journal record into the file. */ - if( rc==SQLITE_OK ){ - void *pData = pPg->pData; - i64 offset = pPager->nSubRec*(4+pPager->pageSize); - char *pData2; - - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); - PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); - rc = write32bits(pPager->sjfd, offset, pPg->pgno); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); + /* Load the entire master journal file into space obtained from + ** sqlite3_malloc() and pointed to by zMasterJournal. Also obtain + ** sufficient space (in zMasterPtr) to hold the names of master + ** journal files extracted from regular rollback-journals. + */ + rc = sqlite3OsFileSize(pMaster, &nMasterJournal); + if( rc!=SQLITE_OK ) goto delmaster_out; + nMasterPtr = pVfs->mxPathname+1; + zMasterJournal = sqlite3Malloc((int)nMasterJournal + nMasterPtr + 1); + if( !zMasterJournal ){ + rc = SQLITE_NOMEM; + goto delmaster_out; + } + zMasterPtr = &zMasterJournal[nMasterJournal+1]; + rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); + if( rc!=SQLITE_OK ) goto delmaster_out; + zMasterJournal[nMasterJournal] = 0; + + zJournal = zMasterJournal; + while( (zJournal-zMasterJournal)nSubRec++; - assert( pPager->nSavepoint>0 ); - rc = addToSavepointBitvecs(pPager, pPg->pgno); + + sqlite3OsClose(pMaster); + rc = sqlite3OsDelete(pVfs, zMaster, 0); + +delmaster_out: + sqlite3_free(zMasterJournal); + if( pMaster ){ + sqlite3OsClose(pMaster); + assert( !isOpen(pJournal) ); + sqlite3_free(pMaster); } return rc; } + /* -** This function is called by the pcache layer when it has reached some -** soft memory limit. The first argument is a pointer to a Pager object -** (cast as a void*). The pager is always 'purgeable' (not an in-memory -** database). The second argument is a reference to a page that is -** currently dirty but has no outstanding references. The page -** is always associated with the Pager object passed as the first -** argument. +** This function is used to change the actual size of the database +** file in the file-system. This only happens when committing a transaction, +** or rolling back a transaction (including rolling back a hot-journal). ** -** The job of this function is to make pPg clean by writing its contents -** out to the database file, if possible. This may involve syncing the -** journal file. +** If the main database file is not open, or the pager is not in either +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS +** xTruncate() method to truncate it. ** -** If successful, sqlite3PcacheMakeClean() is called on the page and -** SQLITE_OK returned. If an IO error occurs while trying to make the -** page clean, the IO error code is returned. If the page cannot be -** made clean for some other reason, but no error occurs, then SQLITE_OK -** is returned by sqlite3PcacheMakeClean() is not called. +** Or, it might might be the case that the file on disk is smaller than +** nPage pages. Some operating system implementations can get confused if +** you try to truncate a file to some size that is larger than it +** currently is, so detect this case and write a single zero byte to +** the end of the new file instead. +** +** If successful, return SQLITE_OK. If an IO error occurs while modifying +** the database file, return the error code to the caller. */ -static int pagerStress(void *p, PgHdr *pPg){ - Pager *pPager = (Pager *)p; +static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; - - assert( pPg->pPager==pPager ); - assert( pPg->flags&PGHDR_DIRTY ); - - /* The doNotSyncSpill flag is set during times when doing a sync of - ** journal (and adding a new header) is not allowed. This occurs - ** during calls to sqlite3PagerWrite() while trying to journal multiple - ** pages belonging to the same sector. - ** - ** The doNotSpill flag inhibits all cache spilling regardless of whether - ** or not a sync is required. This is set during a rollback. - ** - ** Spilling is also prohibited when in an error state since that could - ** lead to database corruption. In the current implementaton it - ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 - ** while in the error state, hence it is impossible for this routine to - ** be called in the error state. Nevertheless, we include a NEVER() - ** test for the error state as a safeguard against future changes. - */ - if( NEVER(pPager->errCode) ) return SQLITE_OK; - if( pPager->doNotSpill ) return SQLITE_OK; - if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ - return SQLITE_OK; - } - - pPg->pDirty = 0; - if( pagerUseWal(pPager) ){ - /* Write a single frame for this page to the log. */ - if( subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); - } - if( rc==SQLITE_OK ){ - rc = pagerWalFrames(pPager, pPg, 0, 0, 0); - } - }else{ - - /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC - || pPager->eState==PAGER_WRITER_CACHEMOD - ){ - rc = syncJournal(pPager, 1); - } - - /* If the page number of this page is larger than the current size of - ** the database image, it may need to be written to the sub-journal. - ** This is because the call to pager_write_pagelist() below will not - ** actually write data to the file in this case. - ** - ** Consider the following sequence of events: - ** - ** BEGIN; - ** - ** - ** SAVEPOINT sp; - ** - ** pagerStress(page X) - ** ROLLBACK TO sp; - ** - ** If (X>Y), then when pagerStress is called page X will not be written - ** out to the database file, but will be dropped from the cache. Then, - ** following the "ROLLBACK TO sp" statement, reading page X will read - ** data from the database file. This will be the copy of page X as it - ** was when the transaction started, not as it was when "SAVEPOINT sp" - ** was executed. - ** - ** The solution is to write the current data for page X into the - ** sub-journal file now (if it is not already there), so that it will - ** be restored to its current value when the "ROLLBACK TO sp" is - ** executed. - */ - if( NEVER( - rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) - ) ){ - rc = subjournalPage(pPg); - } + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); - /* Write the contents of the page out to the database file. */ - if( rc==SQLITE_OK ){ - assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); - rc = pager_write_pagelist(pPager, pPg); + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + i64 currentSize, newSize; + int szPage = pPager->pageSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); + /* TODO: Is it safe to use Pager.dbFileSize here? */ + rc = sqlite3OsFileSize(pPager->fd, ¤tSize); + newSize = szPage*(i64)nPage; + if( rc==SQLITE_OK && currentSize!=newSize ){ + if( currentSize>newSize ){ + rc = sqlite3OsTruncate(pPager->fd, newSize); + }else{ + char *pTmp = pPager->pTmpSpace; + memset(pTmp, 0, szPage); + testcase( (newSize-szPage) < currentSize ); + testcase( (newSize-szPage) == currentSize ); + testcase( (newSize-szPage) > currentSize ); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); + } + if( rc==SQLITE_OK ){ + pPager->dbFileSize = nPage; + } } } + return rc; +} - /* Mark the page as clean. */ - if( rc==SQLITE_OK ){ - PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); - sqlite3PcacheMakeClean(pPg); - } +/* +** Set the value of the Pager.sectorSize variable for the given +** pager based on the value returned by the xSectorSize method +** of the open database file. The sector size will be used used +** to determine the size and alignment of journal header and +** master journal pointers within created journal files. +** +** For temporary files the effective sector size is always 512 bytes. +** +** Otherwise, for non-temporary files, the effective sector size is +** the value returned by the xSectorSize() method rounded up to 32 if +** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it +** is greater than MAX_SECTOR_SIZE. +*/ +static void setSectorSize(Pager *pPager){ + assert( isOpen(pPager->fd) || pPager->tempFile ); - return pager_error(pPager, rc); + if( !pPager->tempFile ){ + /* Sector size doesn't matter for temporary files. Also, the file + ** may not have been opened yet, in which case the OsSectorSize() + ** call will segfault. + */ + pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); + } + if( pPager->sectorSize<32 ){ + pPager->sectorSize = 512; + } + if( pPager->sectorSize>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + pPager->sectorSize = MAX_SECTOR_SIZE; + } } - /* -** Allocate and initialize a new Pager object and put a pointer to it -** in *ppPager. The pager should eventually be freed by passing it -** to sqlite3PagerClose(). +** Playback the journal and thus restore the database file to +** the state it was in before we started making changes. ** -** The zFilename argument is the path to the database file to open. -** If zFilename is NULL then a randomly-named temporary file is created -** and used as the file to be cached. Temporary files are be deleted -** automatically when they are closed. If zFilename is ":memory:" then -** all information is held in cache. It is never written to disk. -** This can be used to implement an in-memory database. +** The journal file format is as follows: ** -** The nExtra parameter specifies the number of bytes of space allocated -** along with each page reference. This space is available to the user -** via the sqlite3PagerGetExtra() API. +** (1) 8 byte prefix. A copy of aJournalMagic[]. +** (2) 4 byte big-endian integer which is the number of valid page records +** in the journal. If this value is 0xffffffff, then compute the +** number of page records from the journal size. +** (3) 4 byte big-endian integer which is the initial value for the +** sanity checksum. +** (4) 4 byte integer which is the number of pages to truncate the +** database to during a rollback. +** (5) 4 byte big-endian integer which is the sector size. The header +** is this many bytes in size. +** (6) 4 byte big-endian integer which is the page size. +** (7) zero padding out to the next sector size. +** (8) Zero or more pages instances, each as follows: +** + 4 byte page number. +** + pPager->pageSize bytes of data. +** + 4 byte checksum ** -** The flags argument is used to specify properties that affect the -** operation of the pager. It should be passed some bitwise combination -** of the PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK flags. +** When we speak of the journal header, we mean the first 7 items above. +** Each entry in the journal is an instance of the 8th item. ** -** The vfsFlags parameter is a bitmask to pass to the flags parameter -** of the xOpen() method of the supplied VFS when opening files. +** Call the value from the second bullet "nRec". nRec is the number of +** valid page entries in the journal. In most cases, you can compute the +** value of nRec from the size of the journal file. But if a power +** failure occurred while the journal was being written, it could be the +** case that the size of the journal file had already been increased but +** the extra entries had not yet made it safely to disk. In such a case, +** the value of nRec computed from the file size would be too large. For +** that reason, we always use the nRec value in the header. ** -** If the pager object is allocated and the specified file opened -** successfully, SQLITE_OK is returned and *ppPager set to point to -** the new pager object. If an error occurs, *ppPager is set to NULL -** and error code returned. This function may return SQLITE_NOMEM -** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or -** various SQLITE_IO_XXX errors. +** If the nRec value is 0xffffffff it means that nRec should be computed +** from the file size. This value is used when the user selects the +** no-sync option for the journal. A power failure could lead to corruption +** in this case. But for things like temporary table (which will be +** deleted when the power is restored) we don't care. +** +** If the file opened as the journal file is not a well-formed +** journal file then all pages up to the first corrupted page are rolled +** back (or no pages if the journal header is corrupted). The journal file +** is then deleted and SQLITE_OK returned, just as if no corruption had +** been encountered. +** +** If an I/O or malloc() error occurs, the journal-file is not deleted +** and an error code is returned. +** +** The isHot parameter indicates that we are trying to rollback a journal +** that might be a hot journal. Or, it could be that the journal is +** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. +** If the journal really is hot, reset the pager cache prior rolling +** back any content. If the journal is merely persistent, no reset is +** needed. */ -SQLITE_PRIVATE int sqlite3PagerOpen( - sqlite3_vfs *pVfs, /* The virtual file system to use */ - Pager **ppPager, /* OUT: Return the Pager structure here */ - const char *zFilename, /* Name of the database file to open */ - int nExtra, /* Extra bytes append to each in-memory page */ - int flags, /* flags controlling this file */ - int vfsFlags, /* flags passed through to sqlite3_vfs.xOpen() */ - void (*xReinit)(DbPage*) /* Function to reinitialize pages */ -){ - u8 *pPtr; - Pager *pPager = 0; /* Pager object to allocate and return */ - int rc = SQLITE_OK; /* Return code */ - int tempFile = 0; /* True for temp files (incl. in-memory files) */ - int memDb = 0; /* True if this is an in-memory file */ - int readOnly = 0; /* True if this is a read-only file */ - int journalFileSize; /* Bytes to allocate for each journal fd */ - char *zPathname = 0; /* Full path to database file */ - int nPathname = 0; /* Number of bytes in zPathname */ - int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ - int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ - int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ - u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ +static int pager_playback(Pager *pPager, int isHot){ + sqlite3_vfs *pVfs = pPager->pVfs; + i64 szJ; /* Size of the journal file in bytes */ + u32 nRec; /* Number of Records in the journal */ + u32 u; /* Unsigned loop counter */ + Pgno mxPg = 0; /* Size of the original file in pages */ + int rc; /* Result code of a subroutine */ + int res = 1; /* Value returned by sqlite3OsAccess() */ + char *zMaster = 0; /* Name of master journal file if any */ + int needPagerReset; /* True to reset page prior to first page rollback */ - /* Figure out how much space is required for each journal file-handle - ** (there are two of them, the main journal and the sub-journal). This - ** is the maximum space required for an in-memory journal file handle - ** and a regular journal file-handle. Note that a "regular journal-handle" - ** may be a wrapper capable of caching the first portion of the journal - ** file in memory to implement the atomic-write optimization (see - ** source file journal.c). + /* Figure out how many records are in the journal. Abort early if + ** the journal is empty. */ - if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ - journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); - }else{ - journalFileSize = ROUND8(sqlite3MemJournalSize()); + assert( isOpen(pPager->jfd) ); + rc = sqlite3OsFileSize(pPager->jfd, &szJ); + if( rc!=SQLITE_OK ){ + goto end_playback; } - /* Set the output variable to NULL in case an error occurs. */ - *ppPager = 0; - -#ifndef SQLITE_OMIT_MEMORYDB - if( flags & PAGER_MEMORY ){ - memDb = 1; - zFilename = 0; + /* Read the master journal name from the journal, if it is present. + ** If a master journal file name is specified, but the file is not + ** present on disk, then the journal is not hot and does not need to be + ** played back. + ** + ** TODO: Technically the following is an error because it assumes that + ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that + ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, + ** mxPathname is 512, which is the same as the minimum allowable value + ** for pageSize. + */ + zMaster = pPager->pTmpSpace; + rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); + if( rc==SQLITE_OK && zMaster[0] ){ + rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); } -#endif + zMaster = 0; + if( rc!=SQLITE_OK || !res ){ + goto end_playback; + } + pPager->journalOff = 0; + needPagerReset = isHot; - /* Compute and store the full pathname in an allocated buffer pointed - ** to by zPathname, length nPathname. Or, if this is a temporary file, - ** leave both nPathname and zPathname set to 0. + /* This loop terminates either when a readJournalHdr() or + ** pager_playback_one_page() call returns SQLITE_DONE or an IO error + ** occurs. */ - if( zFilename && zFilename[0] ){ - nPathname = pVfs->mxPathname+1; - zPathname = sqlite3Malloc(nPathname*2); - if( zPathname==0 ){ - return SQLITE_NOMEM; + while( 1 ){ + /* Read the next journal header from the journal file. If there are + ** not enough bytes left in the journal file for a complete header, or + ** it is corrupted, then a process must have failed while writing it. + ** This indicates nothing more needs to be rolled back. + */ + rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + goto end_playback; } - zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ - rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); - nPathname = sqlite3Strlen30(zPathname); - if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ - /* This branch is taken when the journal path required by - ** the database being opened will be more than pVfs->mxPathname - ** bytes in length. This means the database cannot be opened, - ** as it will not be possible to open the journal file or even - ** check for a hot-journal before reading. - */ - rc = SQLITE_CANTOPEN_BKPT; + + /* If nRec is 0xffffffff, then this journal was created by a process + ** working in no-sync mode. This means that the rest of the journal + ** file consists of pages, there are no more journal headers. Compute + ** the value of nRec based on this assumption. + */ + if( nRec==0xffffffff ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ); + nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager)); } - if( rc!=SQLITE_OK ){ - sqlite3_free(zPathname); - return rc; + + /* If nRec is 0 and this rollback is of a transaction created by this + ** process and if this is the final header in the journal, then it means + ** that this part of the journal was being filled but has not yet been + ** synced to disk. Compute the number of pages based on the remaining + ** size of the file. + ** + ** The third term of the test was added to fix ticket #2565. + ** When rolling back a hot journal, nRec==0 always means that the next + ** chunk of the journal contains zero pages to be rolled back. But + ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in + ** the journal, it means that the journal might contain additional + ** pages that need to be rolled back and that the number of pages + ** should be computed based on the journal file size. + */ + if( nRec==0 && !isHot && + pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ + nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager)); } - } - /* Allocate memory for the Pager structure, PCache object, the - ** three file descriptors, the database file name and the journal - ** file name. The layout in memory is as follows: - ** - ** Pager object (sizeof(Pager) bytes) - ** PCache object (sqlite3PcacheSize() bytes) - ** Database file handle (pVfs->szOsFile bytes) - ** Sub-journal file handle (journalFileSize bytes) - ** Main journal file handle (journalFileSize bytes) - ** Database file name (nPathname+1 bytes) - ** Journal file name (nPathname+8+1 bytes) - */ - pPtr = (u8 *)sqlite3MallocZero( - ROUND8(sizeof(*pPager)) + /* Pager structure */ - ROUND8(pcacheSize) + /* PCache object */ - ROUND8(pVfs->szOsFile) + /* The main db file */ - journalFileSize * 2 + /* The two journal files */ - nPathname + 1 + /* zFilename */ - nPathname + 8 + 1 /* zJournal */ -#ifndef SQLITE_OMIT_WAL - + nPathname + 4 + 1 /* zWal */ -#endif - ); - assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); - if( !pPtr ){ - sqlite3_free(zPathname); - return SQLITE_NOMEM; - } - pPager = (Pager*)(pPtr); - pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); - pPager->fd = (sqlite3_file*)(pPtr += ROUND8(pcacheSize)); - pPager->sjfd = (sqlite3_file*)(pPtr += ROUND8(pVfs->szOsFile)); - pPager->jfd = (sqlite3_file*)(pPtr += journalFileSize); - pPager->zFilename = (char*)(pPtr += journalFileSize); - assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); - - /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ - if( zPathname ){ - assert( nPathname>0 ); - pPager->zJournal = (char*)(pPtr += nPathname + 1); - memcpy(pPager->zFilename, zPathname, nPathname); - memcpy(pPager->zJournal, zPathname, nPathname); - memcpy(&pPager->zJournal[nPathname], "-journal", 8); -#ifndef SQLITE_OMIT_WAL - pPager->zWal = &pPager->zJournal[nPathname+8+1]; - memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal", 4); -#endif - sqlite3_free(zPathname); - } - pPager->pVfs = pVfs; - pPager->vfsFlags = vfsFlags; - - /* Open the pager file. - */ - if( zFilename && zFilename[0] ){ - int fout = 0; /* VFS flags returned by xOpen() */ - rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); - assert( !memDb ); - readOnly = (fout&SQLITE_OPEN_READONLY); + /* If this is the first header read from the journal, truncate the + ** database file back to its original size. + */ + if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ + rc = pager_truncate(pPager, mxPg); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + pPager->dbSize = mxPg; + } - /* If the file was successfully opened for read/write access, - ** choose a default page size in case we have to create the - ** database file. The default page size is the maximum of: - ** - ** + SQLITE_DEFAULT_PAGE_SIZE, - ** + The value returned by sqlite3OsSectorSize() - ** + The largest page size that can be written atomically. + /* Copy original pages out of the journal and back into the + ** database file and/or page cache. */ - if( rc==SQLITE_OK && !readOnly ){ - setSectorSize(pPager); - assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); - if( szPageDfltsectorSize ){ - if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ - szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; - }else{ - szPageDflt = (u32)pPager->sectorSize; - } + for(u=0; ufd); - int ii; - assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); - assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); - assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); - for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ - if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ - szPageDflt = ii; - } + rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + pPager->journalOff = szJ; + break; + }else if( rc==SQLITE_IOERR_SHORT_READ ){ + /* If the journal has been truncated, simply stop reading and + ** processing the journal. This might happen if the journal was + ** not completely written and synced prior to a crash. In that + ** case, the database should have never been written in the + ** first place so it is OK to simply abandon the rollback. */ + rc = SQLITE_OK; + goto end_playback; + }else{ + /* If we are unable to rollback, quit and return the error + ** code. This will cause the pager to enter the error state + ** so that no further harm will be done. Perhaps the next + ** process to come along will be able to rollback the database. + */ + goto end_playback; } } -#endif } - }else{ - /* If a temporary file is requested, it is not opened immediately. - ** In this case we accept the default page size and delay actually - ** opening the file until the first call to OsWrite(). - ** - ** This branch is also run for an in-memory database. An in-memory - ** database is the same as a temp-file that is never written out to - ** disk and uses an in-memory rollback journal. - */ - tempFile = 1; - pPager->eState = PAGER_READER; - pPager->eLock = EXCLUSIVE_LOCK; - readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } + /*NOTREACHED*/ + assert( 0 ); - /* The following call to PagerSetPagesize() serves to set the value of - ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. +end_playback: + /* Following a rollback, the database file should be back in its original + ** state prior to the start of the transaction, so invoke the + ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the + ** assertion that the transaction counter was modified. + */ + assert( + pPager->fd->pMethods==0 || + sqlite3OsFileControl(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0)>=SQLITE_OK + ); + + /* If this playback is happening automatically as a result of an IO or + ** malloc error that occurred after the change-counter was updated but + ** before the transaction was committed, then the change-counter + ** modification may just have been reverted. If this happens in exclusive + ** mode, then subsequent transactions performed by the connection will not + ** update the change-counter at all. This may lead to cache inconsistency + ** problems for other processes at some point in the future. So, just + ** in case this has happened, clear the changeCountDone flag now. */ + pPager->changeCountDone = pPager->tempFile; + if( rc==SQLITE_OK ){ - assert( pPager->memDb==0 ); - rc = sqlite3PagerSetPagesize(pPager, &szPageDflt, -1); + zMaster = pPager->pTmpSpace; + rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } - - /* If an error occurred in either of the blocks above, free the - ** Pager structure and close the file. - */ - if( rc!=SQLITE_OK ){ - assert( !pPager->pTmpSpace ); - sqlite3OsClose(pPager->fd); - sqlite3_free(pPager); - return rc; + if( rc==SQLITE_OK + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + rc = sqlite3PagerSync(pPager); } - - /* Initialize the PCache object. */ - assert( nExtra<1000 ); - nExtra = ROUND8(nExtra); - sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); - - PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); - IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) - - pPager->useJournal = (u8)useJournal; - pPager->noReadlock = (noReadlock && readOnly) ?1:0; - /* pPager->stmtOpen = 0; */ - /* pPager->stmtInUse = 0; */ - /* pPager->nRef = 0; */ - /* pPager->stmtSize = 0; */ - /* pPager->stmtJSize = 0; */ - /* pPager->nPage = 0; */ - pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; - /* pPager->state = PAGER_UNLOCK; */ -#if 0 - assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); -#endif - /* pPager->errMask = 0; */ - pPager->tempFile = (u8)tempFile; - assert( tempFile==PAGER_LOCKINGMODE_NORMAL - || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); - assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); - pPager->exclusiveMode = (u8)tempFile; - pPager->changeCountDone = pPager->tempFile; - pPager->memDb = (u8)memDb; - pPager->readOnly = (u8)readOnly; - assert( useJournal || pPager->tempFile ); - pPager->noSync = pPager->tempFile; - pPager->fullSync = pPager->noSync ?0:1; - pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = pPager->syncFlags; - /* pPager->pFirst = 0; */ - /* pPager->pFirstSynced = 0; */ - /* pPager->pLast = 0; */ - pPager->nExtra = (u16)nExtra; - pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; - assert( isOpen(pPager->fd) || tempFile ); - setSectorSize(pPager); - if( !useJournal ){ - pPager->journalMode = PAGER_JOURNALMODE_OFF; - }else if( memDb ){ - pPager->journalMode = PAGER_JOURNALMODE_MEMORY; + if( rc==SQLITE_OK ){ + rc = pager_end_transaction(pPager, zMaster[0]!='\0'); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK && zMaster[0] && res ){ + /* If there was a master journal and this routine will return success, + ** see if it is possible to delete the master journal. + */ + rc = pager_delmaster(pPager, zMaster); + testcase( rc!=SQLITE_OK ); } - /* pPager->xBusyHandler = 0; */ - /* pPager->pBusyHandlerArg = 0; */ - pPager->xReiniter = xReinit; - /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ - *ppPager = pPager; - return SQLITE_OK; + /* The Pager.sectorSize variable may have been updated while rolling + ** back a journal created by a process with a different sector size + ** value. Reset it to the correct value for this process. + */ + setSectorSize(pPager); + return rc; } - /* -** This function is called after transitioning from PAGER_UNLOCK to -** PAGER_SHARED state. It tests if there is a hot journal present in -** the file-system for the given pager. A hot journal is one that -** needs to be played back. According to this function, a hot-journal -** file exists if the following criteria are met: -** -** * The journal file exists in the file system, and -** * No process holds a RESERVED or greater lock on the database file, and -** * The database file itself is greater than 0 bytes in size, and -** * The first byte of the journal file exists and is not 0x00. -** -** If the current size of the database file is 0 but a journal file -** exists, that is probably an old journal left over from a prior -** database with the same name. In this case the journal file is -** just deleted using OsDelete, *pExists is set to 0 and SQLITE_OK -** is returned. +** Read the content for page pPg out of the database file and into +** pPg->pData. A shared lock or greater must be held on the database +** file before this function is called. ** -** This routine does not check if there is a master journal filename -** at the end of the file. If there is, and that master journal file -** does not exist, then the journal file is not really hot. In this -** case this routine will return a false-positive. The pager_playback() -** routine will discover that the journal file is not really hot and -** will not roll it back. +** If page 1 is read, then the value of Pager.dbFileVers[] is set to +** the value read from the database file. ** -** If a hot-journal file is found to exist, *pExists is set to 1 and -** SQLITE_OK returned. If no hot-journal file is present, *pExists is -** set to 0 and SQLITE_OK returned. If an IO error occurs while trying -** to determine whether or not a hot-journal file exists, the IO error -** code is returned and the value of *pExists is undefined. +** If an IO error occurs, then the IO error is returned to the caller. +** Otherwise, SQLITE_OK is returned. */ -static int hasHotJournal(Pager *pPager, int *pExists){ - sqlite3_vfs * const pVfs = pPager->pVfs; - int rc = SQLITE_OK; /* Return code */ - int exists = 1; /* True if a journal file is present */ - int jrnlOpen = !!isOpen(pPager->jfd); +static int readDbPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ + Pgno pgno = pPg->pgno; /* Page number to read */ + int rc = SQLITE_OK; /* Return code */ + int isInWal = 0; /* True if page is in log file */ + int pgsz = pPager->pageSize; /* Number of bytes to read */ - assert( pPager->useJournal ); + assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); - assert( pPager->eState==PAGER_OPEN ); - - assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & - SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN - )); - *pExists = 0; - if( !jrnlOpen ){ - rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); + if( NEVER(!isOpen(pPager->fd)) ){ + assert( pPager->tempFile ); + memset(pPg->pData, 0, pPager->pageSize); + return SQLITE_OK; } - if( rc==SQLITE_OK && exists ){ - int locked = 0; /* True if some process holds a RESERVED lock */ - /* Race condition here: Another process might have been holding the - ** the RESERVED lock and have a journal open at the sqlite3OsAccess() - ** call above, but then delete the journal and drop the lock before - ** we get to the following sqlite3OsCheckReservedLock() call. If that - ** is the case, this routine might think there is a hot journal when - ** in fact there is none. This results in a false-positive which will - ** be dealt with by the playback routine. Ticket #3883. - */ - rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); - if( rc==SQLITE_OK && !locked ){ - Pgno nPage; /* Number of pages in database file */ + if( pagerUseWal(pPager) ){ + /* Try to pull the page from the write-ahead log. */ + rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); + } + if( rc==SQLITE_OK && !isInWal ){ + i64 iOffset = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } - /* Check the size of the database file. If it consists of 0 pages, - ** then delete the journal file. See the header comment above for - ** the reasoning here. Delete the obsolete journal file under - ** a RESERVED lock to avoid race conditions and to avoid violating - ** [H33020]. + if( pgno==1 ){ + if( rc ){ + /* If the read is unsuccessful, set the dbFileVers[] to something + ** that will never be a valid file version. dbFileVers[] is a copy + ** of bytes 24..39 of the database. Bytes 28..31 should always be + ** zero or the size of the database in page. Bytes 32..35 and 35..39 + ** should be page numbers which are never 0xffffffff. So filling + ** pPager->dbFileVers[] with all 0xff bytes should suffice. + ** + ** For an encrypted database, the situation is more complex: bytes + ** 24..39 of the database are white noise. But the probability of + ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** we should still be ok. */ - rc = pagerPagecount(pPager, &nPage); - if( rc==SQLITE_OK ){ - if( nPage==0 ){ - sqlite3BeginBenignMalloc(); - if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ - sqlite3OsDelete(pVfs, pPager->zJournal, 0); - if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); - } - sqlite3EndBenignMalloc(); - }else{ - /* The journal file exists and no other connection has a reserved - ** or greater lock on the database file. Now check that there is - ** at least one non-zero bytes at the start of the journal file. - ** If there is, then we consider this journal to be hot. If not, - ** it can be ignored. - */ - if( !jrnlOpen ){ - int f = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &f); - } - if( rc==SQLITE_OK ){ - u8 first = 0; - rc = sqlite3OsRead(pPager->jfd, (void *)&first, 1, 0); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } - if( !jrnlOpen ){ - sqlite3OsClose(pPager->jfd); - } - *pExists = (first!=0); - }else if( rc==SQLITE_CANTOPEN ){ - /* If we cannot open the rollback journal file in order to see if - ** its has a zero header, that might be due to an I/O error, or - ** it might be due to the race condition described above and in - ** ticket #3883. Either way, assume that the journal is hot. - ** This might be a false positive. But if it is, then the - ** automatic journal playback and recovery mechanism will deal - ** with it under an EXCLUSIVE lock where we do not need to - ** worry so much with race conditions. - */ - *pExists = 1; - rc = SQLITE_OK; - } - } - } + memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); + }else{ + u8 *dbFileVers = &((u8*)pPg->pData)[24]; + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); + + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pgno)); + PAGERTRACE(("FETCH %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pPg))); return rc; } /* -** This function is called to obtain a shared lock on the database file. -** It is illegal to call sqlite3PagerAcquire() until after this function -** has been successfully called. If a shared-lock is already held when -** this function is called, it is a no-op. -** -** The following operations are also performed by this function. -** -** 1) If the pager is currently in PAGER_OPEN state (no lock held -** on the database file), then an attempt is made to obtain a -** SHARED lock on the database file. Immediately after obtaining -** the SHARED lock, the file-system is checked for a hot-journal, -** which is played back if present. Following any hot-journal -** rollback, the contents of the cache are validated by checking -** the 'change-counter' field of the database file header and -** discarded if they are found to be invalid. -** -** 2) If the pager is running in exclusive-mode, and there are currently -** no outstanding references to any pages, and is in the error state, -** then an attempt is made to clear the error state by discarding -** the contents of the page cache and rolling back any open journal -** file. +** Update the value of the change-counter at offsets 24 and 92 in +** the header and the sqlite version number at offset 96. ** -** If everything is successful, SQLITE_OK is returned. If an IO error -** occurs while locking the database, checking for a hot-journal file or -** rolling back a journal file, the IO error code is returned. +** This is an unconditional update. See also the pager_incr_changecounter() +** routine which only updates the change-counter if the update is actually +** needed, as determined by the pPager->changeCountDone state variable. */ -SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ - int rc = SQLITE_OK; /* Return code */ +static void pager_write_changecounter(PgHdr *pPg){ + u32 change_counter; - /* This routine is only called from b-tree and only when there are no - ** outstanding pages. This implies that the pager state should either - ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - assert( assert_pager_state(pPager) ); - assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } + /* Increment the value just read and write it back to byte 24. */ + change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; + put32bits(((char*)pPg->pData)+24, change_counter); - if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ - int bHotJournal = 1; /* True if there exists a hot journal-file */ + /* Also store the SQLite version number in bytes 96..99 and in + ** bytes 92..95 store the change counter for which the version number + ** is valid. */ + put32bits(((char*)pPg->pData)+92, change_counter); + put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); +} - assert( !MEMDB ); - assert( pPager->noReadlock==0 || pPager->readOnly ); +#ifndef SQLITE_OMIT_WAL +/* +** This function is invoked once for each page that has already been +** written into the log file when a WAL transaction is rolled back. +** Parameter iPg is the page number of said page. The pCtx argument +** is actually a pointer to the Pager structure. +** +** If page iPg is present in the cache, and has no outstanding references, +** it is discarded. Otherwise, if there are one or more outstanding +** references, the page content is reloaded from the database. If the +** attempt to reload content from the database is required and fails, +** return an SQLite error code. Otherwise, SQLITE_OK. +*/ +static int pagerUndoCallback(void *pCtx, Pgno iPg){ + int rc = SQLITE_OK; + Pager *pPager = (Pager *)pCtx; + PgHdr *pPg; - if( pPager->noReadlock==0 ){ - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); - goto failed; + pPg = sqlite3PagerLookup(pPager, iPg); + if( pPg ){ + if( sqlite3PcachePageRefcount(pPg)==1 ){ + sqlite3PcacheDrop(pPg); + }else{ + rc = readDbPage(pPg); + if( rc==SQLITE_OK ){ + pPager->xReiniter(pPg); } + sqlite3PagerUnref(pPg); } + } - /* If a journal file exists, and there is no RESERVED lock on the - ** database file, then it either needs to be played back or deleted. - */ - if( pPager->eLock<=SHARED_LOCK ){ - rc = hasHotJournal(pPager, &bHotJournal); - } - if( rc!=SQLITE_OK ){ - goto failed; - } - if( bHotJournal ){ - /* Get an EXCLUSIVE lock on the database file. At this point it is - ** important that a RESERVED lock is not obtained on the way to the - ** EXCLUSIVE lock. If it were, another process might open the - ** database file, detect the RESERVED lock, and conclude that the - ** database is safe to read while this process is still rolling the - ** hot-journal back. - ** - ** Because the intermediate RESERVED lock is not requested, any - ** other process attempting to access the database file will get to - ** this point in the code and fail to obtain its own EXCLUSIVE lock - ** on the database file. - ** - ** Unless the pager is in locking_mode=exclusive mode, the lock is - ** downgraded to SHARED_LOCK before this function returns. - */ - rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - goto failed; - } - - /* If it is not already open and the file exists on disk, open the - ** journal for read/write access. Write access is required because - ** in exclusive-access mode the file descriptor will be kept open - ** and possibly used for a transaction later on. Also, write-access - ** is usually required to finalize the journal in journal_mode=persist - ** mode (and also for journal_mode=truncate on some systems). - ** - ** If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it - ** may mean that the pager was in the error-state when this - ** function was called and the journal file does not exist. - */ - if( !isOpen(pPager->jfd) ){ - sqlite3_vfs * const pVfs = pPager->pVfs; - int bExists; /* True if journal file exists */ - rc = sqlite3OsAccess( - pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); - if( rc==SQLITE_OK && bExists ){ - int fout = 0; - int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; - assert( !pPager->tempFile ); - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ - rc = SQLITE_CANTOPEN_BKPT; - sqlite3OsClose(pPager->jfd); - } - } - } - - /* Playback and delete the journal. Drop the database write - ** lock and reacquire the read lock. Purge the cache before - ** playing back the hot-journal so that we don't end up with - ** an inconsistent cache. Sync the hot journal before playing - ** it back since the process that crashed and left the hot journal - ** probably did not sync it and we are required to always sync - ** the journal before playing it back. - */ - if( isOpen(pPager->jfd) ){ - assert( rc==SQLITE_OK ); - rc = pagerSyncHotJournal(pPager); - if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); - pPager->eState = PAGER_OPEN; - } - }else if( !pPager->exclusiveMode ){ - pagerUnlockDb(pPager, SHARED_LOCK); - } - - if( rc!=SQLITE_OK ){ - /* This branch is taken if an error occurs while trying to open - ** or roll back a hot-journal while holding an EXCLUSIVE lock. The - ** pager_unlock() routine will be called before returning to unlock - ** the file. If the unlock attempt fails, then Pager.eLock must be - ** set to UNKNOWN_LOCK (see the comment above the #define for - ** UNKNOWN_LOCK above for an explanation). - ** - ** In order to get pager_unlock() to do this, set Pager.eState to - ** PAGER_ERROR now. This is not actually counted as a transition - ** to ERROR state in the state diagram at the top of this file, - ** since we know that the same call to pager_unlock() will very - ** shortly transition the pager object to the OPEN state. Calling - ** assert_pager_state() would fail now, as it should not be possible - ** to be in ERROR state when there are zero outstanding page - ** references. - */ - pager_error(pPager, rc); - goto failed; - } - - assert( pPager->eState==PAGER_OPEN ); - assert( (pPager->eLock==SHARED_LOCK) - || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) - ); - } - - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ - /* The shared-lock has just been acquired on the database file - ** and there are already pages in the cache (from a previous - ** read or write transaction). Check to see if the database - ** has been modified. If the database has changed, flush the - ** cache. - ** - ** Database changes is detected by looking at 15 bytes beginning - ** at offset 24 into the file. The first 4 of these 16 bytes are - ** a 32-bit counter that is incremented with each change. The - ** other bytes change randomly with each file change when - ** a codec is in use. - ** - ** There is a vanishingly small chance that a change will not be - ** detected. The chance of an undetected change is so small that - ** it can be neglected. - */ - Pgno nPage = 0; - char dbFileVers[sizeof(pPager->dbFileVers)]; - - rc = pagerPagecount(pPager, &nPage); - if( rc ) goto failed; - - if( nPage>0 ){ - IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ - goto failed; - } - }else{ - memset(dbFileVers, 0, sizeof(dbFileVers)); - } - - if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ - pager_reset(pPager); - } - } + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + sqlite3BackupRestart(pPager->pBackup); - /* If there is a WAL file in the file-system, open this database in WAL - ** mode. Otherwise, the following function call is a no-op. - */ - rc = pagerOpenWalIfPresent(pPager); -#ifndef SQLITE_OMIT_WAL - assert( pPager->pWal==0 || rc==SQLITE_OK ); -#endif - } + return rc; +} - if( pagerUseWal(pPager) ){ - assert( rc==SQLITE_OK ); - rc = pagerBeginReadTransaction(pPager); - } +/* +** This function is called to rollback a transaction on a WAL database. +*/ +static int pagerRollbackWal(Pager *pPager){ + int rc; /* Return Code */ + PgHdr *pList; /* List of dirty pages to revert */ - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ - rc = pagerPagecount(pPager, &pPager->dbSize); + /* For all pages in the cache that are currently dirty or have already + ** been written (but not committed) to the log file, do one of the + ** following: + ** + ** + Discard the cached page (if refcount==0), or + ** + Reload page content from the database (if refcount>0). + */ + pPager->dbSize = pPager->dbOrigSize; + rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); + pList = sqlite3PcacheDirtyList(pPager->pPCache); + while( pList && rc==SQLITE_OK ){ + PgHdr *pNext = pList->pDirty; + rc = pagerUndoCallback((void *)pPager, pList->pgno); + pList = pNext; } - failed: - if( rc!=SQLITE_OK ){ - assert( !MEMDB ); - pager_unlock(pPager); - assert( pPager->eState==PAGER_OPEN ); - }else{ - pPager->eState = PAGER_READER; - } return rc; } /* -** If the reference count has reached zero, rollback any active -** transaction and unlock the pager. +** This function is a wrapper around sqlite3WalFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. ** -** Except, in locking_mode=EXCLUSIVE when there is nothing to in -** the rollback journal, the unlock is not performed and there is -** nothing to rollback, so this routine is a no-op. +** The list of pages passed into this routine is always sorted by page number. +** Hence, if page 1 appears anywhere on the list, it will be the first page. */ -static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ - pagerUnlockAndRollback(pPager); - } -} - -/* -** Acquire a reference to page number pgno in pager pPager (a page -** reference has type DbPage*). If the requested reference is -** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. -** -** If the requested page is already in the cache, it is returned. -** Otherwise, a new page object is allocated and populated with data -** read from the database file. In some cases, the pcache module may -** choose not to allocate a new page object and may reuse an existing -** object with no outstanding references. -** -** The extra data appended to a page is always initialized to zeros the -** first time a page is loaded into memory. If the page requested is -** already in the cache when this function is called, then the extra -** data is left as it was when the page object was last used. -** -** If the database image is smaller than the requested page or if a -** non-zero value is passed as the noContent parameter and the -** requested page is not already stored in the cache, then no -** actual disk read occurs. In this case the memory image of the -** page is initialized to all zeros. -** -** If noContent is true, it means that we do not care about the contents -** of the page. This occurs in two seperate scenarios: -** -** a) When reading a free-list leaf page from the database, and -** -** b) When a savepoint is being rolled back and we need to load -** a new page into the cache to be filled with the data read -** from the savepoint journal. -** -** If noContent is true, then the data returned is zeroed instead of -** being read from the database. Additionally, the bits corresponding -** to pgno in Pager.pInJournal (bitvec of pages already written to the -** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open -** savepoints are set. This means if the page is made writable at any -** point in the future, using a call to sqlite3PagerWrite(), its contents -** will not be journaled. This saves IO. -** -** The acquisition might fail for several reasons. In all cases, -** an appropriate error code is returned and *ppPage is set to NULL. -** -** See also sqlite3PagerLookup(). Both this routine and Lookup() attempt -** to find a page in the in-memory cache first. If the page is not already -** in memory, this routine goes to disk to read it in whereas Lookup() -** just returns 0. This routine acquires a read-lock the first time it -** has to go to disk, and could also playback an old journal if necessary. -** Since Lookup() never goes to disk, it never has to deal with locks -** or journal files. -*/ -SQLITE_PRIVATE int sqlite3PagerAcquire( - Pager *pPager, /* The pager open on the database file */ - Pgno pgno, /* Page number to fetch */ - DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ +static int pagerWalFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int syncFlags /* Flags to pass to OsSync() (or 0) */ ){ - int rc; - PgHdr *pPg; - - assert( pPager->eState>=PAGER_READER ); - assert( assert_pager_state(pPager) ); - - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } - - /* If the pager is in the error state, return an error immediately. - ** Otherwise, request the page from the PCache layer. */ - if( pPager->errCode!=SQLITE_OK ){ - rc = pPager->errCode; - }else{ - rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); - } + int rc; /* Return code */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) + PgHdr *p; /* For looping over pages */ +#endif - if( rc!=SQLITE_OK ){ - /* Either the call to sqlite3PcacheFetch() returned an error or the - ** pager was already in the error-state when this function was called. - ** Set pPg to 0 and jump to the exception handler. */ - pPg = 0; - goto pager_acquire_err; + assert( pPager->pWal ); +#ifdef SQLITE_DEBUG + /* Verify that the page list is in accending order */ + for(p=pList; p && p->pDirty; p=p->pDirty){ + assert( p->pgno < p->pDirty->pgno ); } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); - - if( (*ppPage)->pPager && !noContent ){ - /* In this case the pcache already contains an initialized copy of - ** the page. Return without further ado. */ - assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - PAGER_INCR(pPager->nHit); - return SQLITE_OK; - - }else{ - /* The pager cache has created a new page. Its content needs to - ** be initialized. */ - - PAGER_INCR(pPager->nMiss); - pPg = *ppPage; - pPg->pPager = pPager; +#endif - /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page - ** number greater than this, or the unused locking-page, is requested. */ - if( pgno>PAGER_MAX_PGNO || pgno==PAGER_MJ_PGNO(pPager) ){ - rc = SQLITE_CORRUPT_BKPT; - goto pager_acquire_err; + if( isCommit ){ + /* If a WAL transaction is being committed, there is no point in writing + ** any pages with page numbers greater than nTruncate into the WAL file. + ** They will never be read by any client. So remove them from the pDirty + ** list here. */ + PgHdr *p; + PgHdr **ppNext = &pList; + for(p=pList; (*ppNext = p); p=p->pDirty){ + if( p->pgno<=nTruncate ) ppNext = &p->pDirty; } + assert( pList ); + } - if( MEMDB || pPager->dbSizefd) ){ - if( pgno>pPager->mxPgno ){ - rc = SQLITE_FULL; - goto pager_acquire_err; - } - if( noContent ){ - /* Failure to set the bits in the InJournal bit-vectors is benign. - ** It merely means that we might do some extra work to journal a - ** page that does not need to be journaled. Nevertheless, be sure - ** to test the case where a malloc error occurs while trying to set - ** a bit in a bit vector. - */ - sqlite3BeginBenignMalloc(); - if( pgno<=pPager->dbOrigSize ){ - TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pgno); - testcase( rc==SQLITE_NOMEM ); - } - TESTONLY( rc = ) addToSavepointBitvecs(pPager, pgno); - testcase( rc==SQLITE_NOMEM ); - sqlite3EndBenignMalloc(); - } - memset(pPg->pData, 0, pPager->pageSize); - IOTRACE(("ZERO %p %d\n", pPager, pgno)); - }else{ - assert( pPg->pPager==pPager ); - rc = readDbPage(pPg); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } + if( pList->pgno==1 ) pager_write_changecounter(pList); + rc = sqlite3WalFrames(pPager->pWal, + pPager->pageSize, pList, nTruncate, isCommit, syncFlags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + PgHdr *p; + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } - pager_set_pagehash(pPg); } - return SQLITE_OK; - -pager_acquire_err: - assert( rc!=SQLITE_OK ); - if( pPg ){ - sqlite3PcacheDrop(pPg); +#ifdef SQLITE_CHECK_PAGES + pList = sqlite3PcacheDirtyList(pPager->pPCache); + for(p=pList; p; p=p->pDirty){ + pager_set_pagehash(p); } - pagerUnlockIfUnused(pPager); +#endif - *ppPage = 0; return rc; } /* -** Acquire a page if it is already in the in-memory cache. Do -** not read the page from disk. Return a pointer to the page, -** or 0 if the page is not in cache. +** Begin a read transaction on the WAL. ** -** See also sqlite3PagerGet(). The difference between this routine -** and sqlite3PagerGet() is that _get() will go to the disk and read -** in the page if the page is not already in cache. This routine -** returns NULL if the page is not in cache or if a disk I/O error -** has ever happened. +** This routine used to be called "pagerOpenSnapshot()" because it essentially +** makes a snapshot of the database at the current point in time and preserves +** that snapshot for use by the reader in spite of concurrently changes by +** other writers or checkpointers. */ -SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ - PgHdr *pPg = 0; - assert( pPager!=0 ); - assert( pgno!=0 ); - assert( pPager->pPCache!=0 ); - assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); - sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); - return pPg; -} +static int pagerBeginReadTransaction(Pager *pPager){ + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ -/* -** Release a page reference. -** -** If the number of references to the page drop to zero, then the -** page is added to the LRU list. When all references to all pages -** are released, a rollback occurs and the lock on the database is -** removed. -*/ -SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ - if( pPg ){ - Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); - pagerUnlockIfUnused(pPager); + assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + + /* sqlite3WalEndReadTransaction() was not called for the previous + ** transaction in locking_mode=EXCLUSIVE. So call it now. If we + ** are in locking_mode=NORMAL and EndRead() was previously called, + ** the duplicate call is harmless. + */ + sqlite3WalEndReadTransaction(pPager->pWal); + + rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); + if( rc!=SQLITE_OK || changed ){ + pager_reset(pPager); } + + return rc; } +#endif /* -** This function is called at the start of every write transaction. -** There must already be a RESERVED or EXCLUSIVE lock on the database -** file when this routine is called. -** -** Open the journal file for pager pPager and write a journal header -** to the start of it. If there are active savepoints, open the sub-journal -** as well. This function is only used when the journal file is being -** opened to write a rollback log for a transaction. It is not used -** when opening a hot journal file to roll it back. -** -** If the journal file is already open (as it may be in exclusive mode), -** then this function just writes a journal header to the start of the -** already open file. -** -** Whether or not the journal file is opened by this function, the -** Pager.pInJournal bitvec structure is allocated. +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). ** -** Return SQLITE_OK if everything is successful. Otherwise, return -** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or -** an IO error code if opening or writing the journal file fails. +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. */ -static int pager_open_journal(Pager *pPager){ - int rc = SQLITE_OK; /* Return code */ - sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ - assert( pPager->eState==PAGER_WRITER_LOCKED ); - assert( assert_pager_state(pPager) ); - assert( pPager->pInJournal==0 ); - - /* If already in the error state, this function is a no-op. But on - ** the other hand, this routine is never called if we are already in - ** an error state. */ - if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + nPage = sqlite3WalDbsize(pPager->pWal); - if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ - pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); - if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; - } - - /* Open the journal file if it is not already open. */ - if( !isOpen(pPager->jfd) ){ - if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ - sqlite3MemJournalOpen(pPager->jfd); - }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); - #ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) - ); - #else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); - #endif + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. + */ + if( nPage==0 ){ + i64 n = 0; /* Size of db file in bytes */ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); } - - - /* Write the first journal header to the journal file and open - ** the sub-journal if necessary. - */ - if( rc==SQLITE_OK ){ - /* TODO: Check if all of these are really required. */ - pPager->nRec = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - rc = writeJournalHdr(pPager); + nPage = (Pgno)(n / pPager->pageSize); + if( nPage==0 && n>0 ){ + nPage = 1; } } - if( rc!=SQLITE_OK ){ - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - }else{ - assert( pPager->eState==PAGER_WRITER_LOCKED ); - pPager->eState = PAGER_WRITER_CACHEMOD; + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; } - return rc; + *pnPage = nPage; + return SQLITE_OK; } +#ifndef SQLITE_OMIT_WAL /* -** Begin a write-transaction on the specified pager object. If a -** write-transaction has already been opened, this function is a no-op. +** Check if the *-wal file that corresponds to the database opened by pPager +** exists if the database is not empy, or verify that the *-wal file does +** not exist (by deleting it) if the database file is empty. ** -** If the exFlag argument is false, then acquire at least a RESERVED -** lock on the database file. If exFlag is true, then acquire at least -** an EXCLUSIVE lock. If such a lock is already held, no locking -** functions need be called. +** If the database is not empty and the *-wal file exists, open the pager +** in WAL mode. If the database is empty or if no *-wal file exists and +** if no error occurs, make sure Pager.journalMode is not set to +** PAGER_JOURNALMODE_WAL. ** -** If the subjInMemory argument is non-zero, then any sub-journal opened -** within this transaction will be opened as an in-memory file. This -** has no effect if the sub-journal is already opened (as it may be when -** running in exclusive mode) or if the transaction does not require a -** sub-journal. If the subjInMemory argument is zero, then any required -** sub-journal is implemented in-memory if pPager is an in-memory database, -** or using a temporary file otherwise. +** Return SQLITE_OK or an error code. +** +** The caller must hold a SHARED lock on the database file to call this +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL on a none-empty database, this ensures there is no race condition +** between the xAccess() below and an xDelete() being executed by some +** other connection. */ -SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ +static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); - if( pPager->errCode ) return pPager->errCode; - assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - - if( ALWAYS(pPager->eState==PAGER_READER) ){ - assert( pPager->pInJournal==0 ); - - if( pagerUseWal(pPager) ){ - /* If the pager is configured to use locking_mode=exclusive, and an - ** exclusive lock on the database is not already held, obtain it now. - */ - if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - return rc; - } - sqlite3WalExclusiveMode(pPager->pWal, 1); - } + if( !pPager->tempFile ){ + int isWal; /* True if WAL file exists */ + Pgno nPage; /* Size of the database file */ - /* Grab the write lock on the log file. If successful, upgrade to - ** PAGER_RESERVED state. Otherwise, return an error code to the caller. - ** The busy-handler is not invoked if another connection already - ** holds the write-lock. If possible, the upper layer will call it. - */ - rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + rc = pagerPagecount(pPager, &nPage); + if( rc ) return rc; + if( nPage==0 ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + isWal = 0; }else{ - /* Obtain a RESERVED lock on the database file. If the exFlag parameter - ** is true, then immediately upgrade this to an EXCLUSIVE lock. The - ** busy-handler callback can be used when upgrading to the EXCLUSIVE - ** lock, but not when obtaining the RESERVED lock. - */ - rc = pagerLockDb(pPager, RESERVED_LOCK); - if( rc==SQLITE_OK && exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal + ); } - if( rc==SQLITE_OK ){ - /* Change to WRITER_LOCKED state. - ** - ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD - ** when it has an open transaction, but never to DBMOD or FINISHED. - ** This is because in those states the code to roll back savepoint - ** transactions may copy data from the sub-journal into the database - ** file as well as into the page cache. Which would be incorrect in - ** WAL mode. - */ - pPager->eState = PAGER_WRITER_LOCKED; - pPager->dbHintSize = pPager->dbSize; - pPager->dbFileSize = pPager->dbSize; - pPager->dbOrigSize = pPager->dbSize; - pPager->journalOff = 0; + if( isWal ){ + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); + rc = sqlite3PagerOpenWal(pPager, 0); + }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + pPager->journalMode = PAGER_JOURNALMODE_DELETE; + } } - - assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); - assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); - assert( assert_pager_state(pPager) ); } - - PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); return rc; } +#endif /* -** Mark a single data page as writeable. The page is written into the -** main journal or sub-journal as required. If the page is written into -** one of the journals, the corresponding bit is set in the -** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs -** of any open savepoints as appropriate. +** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback +** the entire master journal file. The case pSavepoint==NULL occurs when +** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction +** savepoint. +** +** When pSavepoint is not NULL (meaning a non-transaction savepoint is +** being rolled back), then the rollback consists of up to three stages, +** performed in the order specified: +** +** * Pages are played back from the main journal starting at byte +** offset PagerSavepoint.iOffset and continuing to +** PagerSavepoint.iHdrOffset, or to the end of the main journal +** file if PagerSavepoint.iHdrOffset is zero. +** +** * If PagerSavepoint.iHdrOffset is not zero, then pages are played +** back starting from the journal header immediately following +** PagerSavepoint.iHdrOffset to the end of the main journal file. +** +** * Pages are then played back from the sub-journal file, starting +** with the PagerSavepoint.iSubRec and continuing to the end of +** the journal file. +** +** Throughout the rollback process, each time a page is rolled back, the +** corresponding bit is set in a bitvec structure (variable pDone in the +** implementation below). This is used to ensure that a page is only +** rolled back the first time it is encountered in either journal. +** +** If pSavepoint is NULL, then pages are only played back from the main +** journal file. There is no need for a bitvec in this case. +** +** In either case, before playback commences the Pager.dbSize variable +** is reset to the value that it held at the start of the savepoint +** (or transaction). No page with a page-number greater than this value +** is played back. If one is encountered it is simply skipped. */ -static int pager_write(PgHdr *pPg){ - void *pData = pPg->pData; - Pager *pPager = pPg->pPager; - int rc = SQLITE_OK; +static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ + i64 szJ; /* Effective size of the main journal */ + i64 iHdrOff; /* End of first segment of main-journal records */ + int rc = SQLITE_OK; /* Return code */ + Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ - /* This routine is not called unless a write-transaction has already - ** been started. The journal file may or may not be open at this point. - ** It is never called in the ERROR state. - */ - assert( pPager->eState==PAGER_WRITER_LOCKED - || pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - ); - assert( assert_pager_state(pPager) ); + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); - /* If an error has been previously detected, report the same error - ** again. This should not happen, but the check provides robustness. */ - if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Allocate a bitvec to use to store the set of pages rolled back */ + if( pSavepoint ){ + pDone = sqlite3BitvecCreate(pSavepoint->nOrig); + if( !pDone ){ + return SQLITE_NOMEM; + } + } - /* Higher-level routines never call this function if database is not - ** writable. But check anyway, just for robustness. */ - if( NEVER(pPager->readOnly) ) return SQLITE_PERM; + /* Set the database size back to the value it was before the savepoint + ** being reverted was opened. + */ + pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; + pPager->changeCountDone = pPager->tempFile; - CHECK_PAGE(pPg); + if( !pSavepoint && pagerUseWal(pPager) ){ + return pagerRollbackWal(pPager); + } - /* The journal file needs to be opened. Higher level routines have already - ** obtained the necessary locks to begin the write-transaction, but the - ** rollback journal might not yet be open. Open it now if this is the case. - ** - ** This is done before calling sqlite3PcacheMakeDirty() on the page. - ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then - ** an error might occur and the pager would end up in WRITER_LOCKED state - ** with pages marked as dirty in the cache. + /* Use pPager->journalOff as the effective size of the main rollback + ** journal. The actual file might be larger than this in + ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything + ** past pPager->journalOff is off-limits to us. */ - if( pPager->eState==PAGER_WRITER_LOCKED ){ - rc = pager_open_journal(pPager); - if( rc!=SQLITE_OK ) return rc; - } - assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); - assert( assert_pager_state(pPager) ); + szJ = pPager->journalOff; + assert( pagerUseWal(pPager)==0 || szJ==0 ); - /* Mark the page as dirty. If the page has already been written - ** to the journal then we can return right away. + /* Begin by rolling back records from the main journal starting at + ** PagerSavepoint.iOffset and continuing to the next journal header. + ** There might be records in the main journal that have a page number + ** greater than the current database size (pPager->dbSize) but those + ** will be skipped automatically. Pages are added to pDone as they + ** are played back. */ - sqlite3PcacheMakeDirty(pPg); - if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ - assert( !pagerUseWal(pPager) ); + if( pSavepoint && !pagerUseWal(pPager) ){ + iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; + pPager->journalOff = pSavepoint->iOffset; + while( rc==SQLITE_OK && pPager->journalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); }else{ - - /* The transaction journal now exists and we have a RESERVED or an - ** EXCLUSIVE lock on the main database file. Write the current page to - ** the transaction journal if it is not there already. - */ - if( !pageInJournal(pPg) && !pagerUseWal(pPager) ){ - assert( pagerUseWal(pPager)==0 ); - if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ - u32 cksum; - char *pData2; - i64 iOff = pPager->journalOff; - - /* We should never write to the journal file the page that - ** contains the database locks. The following assert verifies - ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - - assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); - cksum = pager_cksum(pPager, (u8*)pData2); - - /* Even if an IO or diskfull error occurs while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - pPg->flags |= PGHDR_NEED_SYNC; - - rc = write32bits(pPager->jfd, iOff, pPg->pgno); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); - if( rc!=SQLITE_OK ) return rc; - rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); - if( rc!=SQLITE_OK ) return rc; + pPager->journalOff = 0; + } - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, pPager->pageSize)); - PAGER_INCR(sqlite3_pager_writej_count); - PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + /* Continue rolling back records out of the main journal starting at + ** the first journal header seen and continuing until the effective end + ** of the main journal file. Continue to skip out-of-range pages and + ** continue adding pages rolled back to pDone. + */ + while( rc==SQLITE_OK && pPager->journalOffjournalOff += 8 + pPager->pageSize; - pPager->nRec++; - assert( pPager->pInJournal!=0 ); - rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - testcase( rc==SQLITE_NOMEM ); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - rc |= addToSavepointBitvecs(pPager, pPg->pgno); - if( rc!=SQLITE_OK ){ - assert( rc==SQLITE_NOMEM ); - return rc; - } - }else{ - if( pPager->eState!=PAGER_WRITER_DBMOD ){ - pPg->flags |= PGHDR_NEED_SYNC; - } - PAGERTRACE(("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); - } - } - - /* If the statement journal is open and the page is not in it, - ** then write the current page to the statement journal. Note that - ** the statement journal format differs from the standard journal format - ** in that it omits the checksums and the header. + /* + ** The "pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff" + ** test is related to ticket #2565. See the discussion in the + ** pager_playback() function for additional information. */ - if( subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); + if( nJRec==0 + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff + ){ + nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); + } + for(ii=0; rc==SQLITE_OK && iijournalOffjournalOff, pDone, 1, 1); } + assert( rc!=SQLITE_DONE ); } + assert( rc!=SQLITE_OK || pPager->journalOff>=szJ ); - /* Update the database size and return. + /* Finally, rollback pages from the sub-journal. Page that were + ** previously rolled back out of the main journal (and are hence in pDone) + ** will be skipped. Out-of-range pages are also skipped. */ - if( pPager->dbSizepgno ){ - pPager->dbSize = pPg->pgno; - } - return rc; -} - -/* -** Mark a data page as writeable. This routine must be called before -** making changes to a page. The caller must check the return value -** of this function and be careful not to change any page data unless -** this routine returns SQLITE_OK. -** -** The difference between this function and pager_write() is that this -** function also deals with the special case where 2 or more pages -** fit on a single disk sector. In this case all co-resident pages -** must have been written to the journal file before returning. -** -** If an error occurs, SQLITE_NOMEM or an IO error code is returned -** as appropriate. Otherwise, SQLITE_OK. -*/ -SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ - int rc = SQLITE_OK; - - PgHdr *pPg = pDbPage; - Pager *pPager = pPg->pPager; - Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - - assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); - assert( assert_pager_state(pPager) ); - - if( nPagePerSector>1 ){ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - - /* Set the doNotSyncSpill flag to 1. This is because we cannot allow - ** a journal header to be written between the pages journaled by - ** this function. - */ - assert( !MEMDB ); - assert( pPager->doNotSyncSpill==0 ); - pPager->doNotSyncSpill++; - - /* This trick assumes that both the page-size and sector-size are - ** an integer power of 2. It sets variable pg1 to the identifier - ** of the first page of the sector pPg is located on. - */ - pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - - nPageCount = pPager->dbSize; - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); + if( pSavepoint ){ + u32 ii; /* Loop counter */ + i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); - for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ - if( pg!=PAGER_MJ_PGNO(pPager) ){ - rc = sqlite3PagerGet(pPager, pg, &pPage); - if( rc==SQLITE_OK ){ - rc = pager_write(pPage); - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnref(pPage); - } - } - }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnref(pPage); - } + if( pagerUseWal(pPager) ){ + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); } - - /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages - ** starting at pg1, then it needs to be set for all of them. Because - ** writing to any of these nPage pages may damage the others, the - ** journal file must contain sync()ed copies of all of them - ** before any of them can be written out to the database file. - */ - if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB ); - for(ii=0; iiflags |= PGHDR_NEED_SYNC; - sqlite3PagerUnref(pPage); - } - } + for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ + assert( offset==ii*(4+pPager->pageSize) ); + rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); } + assert( rc!=SQLITE_DONE ); + } - assert( pPager->doNotSyncSpill==1 ); - pPager->doNotSyncSpill--; - }else{ - rc = pager_write(pDbPage); + sqlite3BitvecDestroy(pDone); + if( rc==SQLITE_OK ){ + pPager->journalOff = szJ; } + return rc; } /* -** Return TRUE if the page given in the argument was previously passed -** to sqlite3PagerWrite(). In other words, return TRUE if it is ok -** to change the content of the page. +** Change the maximum number of in-memory pages that are allowed. */ -#ifndef NDEBUG -SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->flags&PGHDR_DIRTY; +SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ + sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } -#endif /* -** A call to this routine tells the pager that it is not necessary to -** write the information on page pPg back to the disk, even though -** that page might be marked as dirty. This happens, for example, when -** the page has been added as a leaf of the freelist and so its -** content no longer matters. +** Adjust the robustness of the database to damage due to OS crashes +** or power failures by changing the number of syncs()s when writing +** the rollback journal. There are three levels: ** -** The overlying software layer calls this routine when all of the data -** on the given page is unused. The pager marks the page as clean so -** that it does not get written to disk. +** OFF sqlite3OsSync() is never called. This is the default +** for temporary and transient files. ** -** Tests show that this optimization can quadruple the speed of large -** DELETE operations. +** NORMAL The journal is synced once before writes begin on the +** database. This is normally adequate protection, but +** it is theoretically possible, though very unlikely, +** that an inopertune power failure could leave the journal +** in a state which would cause damage to the database +** when it is rolled back. +** +** FULL The journal is synced twice before writes begin on the +** database (with some additional information - the nRec field +** of the journal header - being written in between the two +** syncs). If we assume that writing a +** single disk sector is atomic, then this mode provides +** assurance that the journal will not be corrupted to the +** point of causing damage to the database during rollback. +** +** The above is for a rollback-journal mode. For WAL mode, OFF continues +** to mean that no syncs ever occur. NORMAL means that the WAL is synced +** prior to the start of checkpoint and that the database file is synced +** at the conclusion of the checkpoint if the entire content of the WAL +** was written back into the database. But no sync operations occur for +** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL +** file is synced following each commit operation, in addition to the +** syncs associated with NORMAL. +** +** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The +** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync +** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an +** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL +** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the +** synchronous=FULL versus synchronous=NORMAL setting determines when +** the xSync primitive is called and is relevant to all platforms. +** +** Numeric values associated with these states are OFF==1, NORMAL=2, +** and FULL=3. */ -SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ - PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); - IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) - pPg->flags |= PGHDR_DONT_WRITE; - pager_set_pagehash(pPg); +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel( + Pager *pPager, /* The pager to set safety level for */ + int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ + int bFullFsync, /* PRAGMA fullfsync */ + int bCkptFullFsync /* PRAGMA checkpoint_fullfsync */ +){ + assert( level>=1 && level<=3 ); + pPager->noSync = (level==1 || pPager->tempFile) ?1:0; + pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; + if( pPager->noSync ){ + pPager->syncFlags = 0; + pPager->ckptSyncFlags = 0; + }else if( bFullFsync ){ + pPager->syncFlags = SQLITE_SYNC_FULL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else if( bCkptFullFsync ){ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else{ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; } } +#endif /* -** This routine is called to increment the value of the database file -** change-counter, stored as a 4-byte big-endian integer starting at -** byte offset 24 of the pager file. -** -** If the isDirectMode flag is zero, then this is done by calling -** sqlite3PagerWrite() on page 1, then modifying the contents of the -** page data. In this case the file will be updated when the current -** transaction is committed. -** -** The isDirectMode flag may only be non-zero if the library was compiled -** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, -** if isDirect is non-zero, then the database file is updated directly -** by writing an updated version of page 1 using a call to the -** sqlite3OsWrite() function. +** The following global variable is incremented whenever the library +** attempts to open a temporary file. This information is used for +** testing and analysis only. */ -static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ - int rc = SQLITE_OK; +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_opentemp_count = 0; +#endif - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - ); - assert( assert_pager_state(pPager) ); +/* +** Open a temporary file. +** +** Write the file descriptor into *pFile. Return SQLITE_OK on success +** or some other error code if we fail. The OS will automatically +** delete the temporary file when it is closed. +** +** The flags passed to the VFS layer xOpen() call are those specified +** by parameter vfsFlags ORed with the following: +** +** SQLITE_OPEN_READWRITE +** SQLITE_OPEN_CREATE +** SQLITE_OPEN_EXCLUSIVE +** SQLITE_OPEN_DELETEONCLOSE +*/ +static int pagerOpentemp( + Pager *pPager, /* The pager object */ + sqlite3_file *pFile, /* Write the file descriptor here */ + int vfsFlags /* Flags passed through to the VFS */ +){ + int rc; /* Return code */ - /* Declare and initialize constant integer 'isDirect'. If the - ** atomic-write optimization is enabled in this build, then isDirect - ** is initialized to the value passed as the isDirectMode parameter - ** to this function. Otherwise, it is always set to zero. - ** - ** The idea is that if the atomic-write optimization is not - ** enabled at compile time, the compiler can omit the tests of - ** 'isDirect' below, as well as the block enclosed in the - ** "if( isDirect )" condition. - */ -#ifndef SQLITE_ENABLE_ATOMIC_WRITE -# define DIRECT_MODE 0 - assert( isDirectMode==0 ); - UNUSED_PARAMETER(isDirectMode); -#else -# define DIRECT_MODE isDirectMode +#ifdef SQLITE_TEST + sqlite3_opentemp_count++; /* Used for testing and analysis only */ #endif - if( !pPager->changeCountDone && pPager->dbSize>0 ){ - PgHdr *pPgHdr; /* Reference to page 1 */ - u32 change_counter; /* Initial value of change-counter field */ + vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); + assert( rc!=SQLITE_OK || isOpen(pFile) ); + return rc; +} - assert( !pPager->tempFile && isOpen(pPager->fd) ); +/* +** Set the busy handler function. +** +** The pager invokes the busy-handler if sqlite3OsLock() returns +** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, +** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE +** lock. It does *not* invoke the busy handler when upgrading from +** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE +** (which occurs during hot-journal rollback). Summary: +** +** Transition | Invokes xBusyHandler +** -------------------------------------------------------- +** NO_LOCK -> SHARED_LOCK | Yes +** SHARED_LOCK -> RESERVED_LOCK | No +** SHARED_LOCK -> EXCLUSIVE_LOCK | No +** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes +** +** If the busy-handler callback returns non-zero, the lock is +** retried. If it returns zero, then the SQLITE_BUSY error is +** returned to the caller of the pager API function. +*/ +SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( + Pager *pPager, /* Pager object */ + int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ + void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ +){ + pPager->xBusyHandler = xBusyHandler; + pPager->pBusyHandlerArg = pBusyHandlerArg; +} - /* Open page 1 of the file for writing. */ - rc = sqlite3PagerGet(pPager, 1, &pPgHdr); - assert( pPgHdr==0 || rc==SQLITE_OK ); +/* +** Change the page size used by the Pager object. The new page size +** is passed in *pPageSize. +** +** If the pager is in the error state when this function is called, it +** is a no-op. The value returned is the error state error code (i.e. +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). +** +** Otherwise, if all of the following are true: +** +** * the new page size (value of *pPageSize) is valid (a power +** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and +** +** * there are no outstanding page references, and +** +** * the database is either not an in-memory database or it is +** an in-memory database that currently consists of zero pages. +** +** then the pager object page size is set to *pPageSize. +** +** If the page size is changed, then this function uses sqlite3PagerMalloc() +** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt +** fails, SQLITE_NOMEM is returned and the page size remains unchanged. +** In all other cases, SQLITE_OK is returned. +** +** If the page size is not changed, either because one of the enumerated +** conditions above is not true, the pager was in error state when this +** function was called, or because the memory allocation attempt failed, +** then *pPageSize is set to the old, retained page size before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; - /* If page one was fetched successfully, and this function is not - ** operating in direct-mode, make page 1 writable. When not in - ** direct mode, page 1 is always held in cache and hence the PagerGet() - ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. - */ - if( !DIRECT_MODE && ALWAYS(rc==SQLITE_OK) ){ - rc = sqlite3PagerWrite(pPgHdr); - } + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ - if( rc==SQLITE_OK ){ - /* Increment the value just read and write it back to byte 24. */ - change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers); - change_counter++; - put32bits(((char*)pPgHdr->pData)+24, change_counter); + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; - /* Also store the SQLite version number in bytes 96..99 and in - ** bytes 92..95 store the change counter for which the version number - ** is valid. */ - put32bits(((char*)pPgHdr->pData)+92, change_counter); - put32bits(((char*)pPgHdr->pData)+96, SQLITE_VERSION_NUMBER); + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); + } + if( rc==SQLITE_OK ){ + pNew = (char *)sqlite3PageMalloc(pageSize); + if( !pNew ) rc = SQLITE_NOMEM; + } - /* If running in direct mode, write the contents of page 1 to the file. */ - if( DIRECT_MODE ){ - const void *zBuf; - assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); - } - if( rc==SQLITE_OK ){ - pPager->changeCountDone = 1; - } - }else{ - pPager->changeCountDone = 1; - } + if( rc==SQLITE_OK ){ + pager_reset(pPager); + pPager->dbSize = (Pgno)(nByte/pageSize); + pPager->pageSize = pageSize; + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); } + } - /* Release the page reference. */ - sqlite3PagerUnref(pPgHdr); + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ + if( nReserve<0 ) nReserve = pPager->nReserve; + assert( nReserve>=0 && nReserve<1000 ); + pPager->nReserve = (i16)nReserve; + pagerReportSize(pPager); } return rc; } /* -** Sync the database file to disk. This is a no-op for in-memory databases -** or pages with the Pager.noSync flag set. +** Return a pointer to the "temporary page" buffer held internally +** by the pager. This is a buffer that is big enough to hold the +** entire content of a database page. This buffer is used internally +** during rollback and will be overwritten whenever a rollback +** occurs. But other modules are free to use it too, as long as +** no rollbacks are happening. +*/ +SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager *pPager){ + return pPager->pTmpSpace; +} + +/* +** Attempt to set the maximum database page count if mxPage is positive. +** Make no changes if mxPage is zero or negative. And never reduce the +** maximum page count below the current size of the database. ** -** If successful, or if called on a pager for which it is a no-op, this -** function returns SQLITE_OK. Otherwise, an IO error code is returned. +** Regardless of mxPage, return the current maximum page count. */ -SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){ - int rc; /* Return code */ - assert( !MEMDB ); - if( pPager->noSync ){ - rc = SQLITE_OK; - }else{ - rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); +SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ + if( mxPage>0 ){ + pPager->mxPgno = mxPage; } - return rc; + assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ + assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ + return pPager->mxPgno; } /* -** This function may only be called while a write-transaction is active in -** rollback. If the connection is in WAL mode, this call is a no-op. -** Otherwise, if the connection does not already have an EXCLUSIVE lock on -** the database file, an attempt is made to obtain one. +** The following set of routines are used to disable the simulated +** I/O error mechanism. These routines are used to avoid simulated +** errors in places where we do not care about errors. ** -** If the EXCLUSIVE lock is already held or the attempt to obtain it is -** successful, or the connection is in WAL mode, SQLITE_OK is returned. -** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is -** returned. +** Unless -DSQLITE_TEST=1 is used, these routines are all no-ops +** and generate no code. */ -SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ +#ifdef SQLITE_TEST +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_hit; +static int saved_cnt; +void disable_simulated_io_errors(void){ + saved_cnt = sqlite3_io_error_pending; + sqlite3_io_error_pending = -1; +} +void enable_simulated_io_errors(void){ + sqlite3_io_error_pending = saved_cnt; +} +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +/* +** Read the first N bytes from the beginning of the file into memory +** that pDest points to. +** +** If the pager was opened on a transient file (zFilename==""), or +** opened on a file less than N bytes in size, the output buffer is +** zeroed and SQLITE_OK returned. The rationale for this is that this +** function is used to read database headers, and a new transient or +** zero sized database has a header than consists entirely of zeroes. +** +** If any IO error apart from SQLITE_IOERR_SHORT_READ is encountered, +** the error code is returned to the caller and the contents of the +** output buffer undefined. +*/ +SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ int rc = SQLITE_OK; - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED - ); - assert( assert_pager_state(pPager) ); - if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + memset(pDest, 0, N); + assert( isOpen(pPager->fd) || pPager->tempFile ); + + /* This routine is only called by btree immediately after creating + ** the Pager object. There has not been an opportunity to transition + ** to WAL mode yet. + */ + assert( !pagerUseWal(pPager) ); + + if( isOpen(pPager->fd) ){ + IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) + rc = sqlite3OsRead(pPager->fd, pDest, N, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } } return rc; } /* -** Sync the database file for the pager pPager. zMaster points to the name -** of a master journal file that should be written into the individual -** journal file. zMaster may be NULL, which is interpreted as no master -** journal (a single database transaction). +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. ** -** This routine ensures that: +** However, if the file is between 1 and bytes in size, then +** this is considered a 1 page file. +*/ +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; +} + + +/* +** Try to obtain a lock of type locktype on the database file. If +** a similar or greater lock is already held, this function is a no-op +** (returning SQLITE_OK immediately). ** -** * The database file change-counter is updated, -** * the journal is synced (unless the atomic-write optimization is used), -** * all dirty pages are written to the database file, -** * the database file is truncated (if required), and -** * the database file synced. +** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke +** the busy callback if the lock is currently not available. Repeat +** until the busy callback returns false or until the attempt to +** obtain the lock succeeds. ** -** The only thing that remains to commit the transaction is to finalize -** (delete, truncate or zero the first part of) the journal file (or -** delete the master journal file if specified). +** Return SQLITE_OK on success and an error code if we cannot obtain +** the lock. If the lock is obtained successfully, set the Pager.state +** variable to locktype before returning. +*/ +static int pager_wait_on_lock(Pager *pPager, int locktype){ + int rc; /* Return code */ + + /* Check that this is either a no-op (because the requested lock is + ** already held, or one of the transistions that the busy-handler + ** may be invoked during, according to the comment above + ** sqlite3PagerSetBusyhandler(). + */ + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) + ); + + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + return rc; +} + +/* +** Function assertTruncateConstraint(pPager) checks that one of the +** following is true for all dirty pages currently in the page-cache: ** -** Note that if zMaster==NULL, this does not overwrite a previous value -** passed to an sqlite3PagerCommitPhaseOne() call. +** a) The page number is less than or equal to the size of the +** current database image, in pages, OR ** -** If the final parameter - noSync - is true, then the database file itself -** is not synced. The caller must call sqlite3PagerSync() directly to -** sync the database file before calling CommitPhaseTwo() to delete the -** journal file in this case. +** b) if the page content were written at this time, it would not +** be necessary to write the current content out to the sub-journal +** (as determined by function subjRequiresPage()). +** +** If the condition asserted by this function were not true, and the +** dirty page were to be discarded from the cache via the pagerStress() +** routine, pagerStress() would not write the current page content to +** the database file. If a savepoint transaction were rolled back after +** this happened, the correct behaviour would be to restore the current +** content of the page. However, since this content is not present in either +** the database file or the portion of the rollback journal and +** sub-journal rolled back the content could not be restored and the +** database image would become corrupt. It is therefore fortunate that +** this circumstance cannot arise. */ -SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( - Pager *pPager, /* Pager object */ - const char *zMaster, /* If not NULL, the master journal name */ - int noSync /* True to omit the xSync on the db file */ -){ - int rc = SQLITE_OK; /* Return code */ +#if defined(SQLITE_DEBUG) +static void assertTruncateConstraintCb(PgHdr *pPg){ + assert( pPg->flags&PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) || pPg->pgno<=pPg->pPager->dbSize ); +} +static void assertTruncateConstraint(Pager *pPager){ + sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); +} +#else +# define assertTruncateConstraint(pPager) +#endif - assert( pPager->eState==PAGER_WRITER_LOCKED - || pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_ERROR - ); - assert( assert_pager_state(pPager) ); +/* +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the +** truncation will be done when the current transaction is committed. +*/ +SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ + assert( pPager->dbSize>=nPage ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + pPager->dbSize = nPage; + assertTruncateConstraint(pPager); +} - /* If a prior error occurred, report that error again. */ - if( NEVER(pPager->errCode) ) return pPager->errCode; - PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", - pPager->zFilename, zMaster, pPager->dbSize)); +/* +** This function is called before attempting a hot-journal rollback. It +** syncs the journal file to disk, then sets pPager->journalHdr to the +** size of the journal file so that the pager_playback() routine knows +** that the entire journal file has been synced. +** +** Syncing a hot-journal to disk before attempting to roll it back ensures +** that if a power-failure occurs during the rollback, the process that +** attempts rollback following system recovery sees the same journal +** content as this process. +** +** If everything goes as planned, SQLITE_OK is returned. Otherwise, +** an SQLite error code. +*/ +static int pagerSyncHotJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_NORMAL); + } + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileSize(pPager->jfd, &pPager->journalHdr); + } + return rc; +} - /* If no database changes have been made, return early. */ - if( pPager->eStatepTmpSpace; + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + /* pPager->errCode = 0; */ + pPager->exclusiveMode = 0; +#ifndef SQLITE_OMIT_WAL + sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); + pPager->pWal = 0; +#endif + pager_reset(pPager); if( MEMDB ){ - /* If this is an in-memory db, or no pages have been written to, or this - ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ - sqlite3BackupRestart(pPager->pBackup); + pager_unlock(pPager); }else{ - if( pagerUseWal(pPager) ){ - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); - if( pList ){ - rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, - (pPager->fullSync ? pPager->syncFlags : 0) - ); - } - if( rc==SQLITE_OK ){ - sqlite3PcacheCleanAll(pPager->pPCache); - } - }else{ - /* The following block updates the change-counter. Exactly how it - ** does this depends on whether or not the atomic-update optimization - ** was enabled at compile time, and if this transaction meets the - ** runtime criteria to use the operation: - ** - ** * The file-system supports the atomic-write property for - ** blocks of size page-size, and - ** * This commit is not part of a multi-file transaction, and - ** * Exactly one page has been modified and store in the journal file. - ** - ** If the optimization was not enabled at compile time, then the - ** pager_incr_changecounter() function is called to update the change - ** counter in 'indirect-mode'. If the optimization is compiled in but - ** is not applicable to this transaction, call sqlite3JournalCreate() - ** to make sure the journal file has actually been created, then call - ** pager_incr_changecounter() to update the change-counter in indirect - ** mode. - ** - ** Otherwise, if the optimization is both enabled and applicable, - ** then call pager_incr_changecounter() to update the change-counter - ** in 'direct' mode. In this case the journal file will never be - ** created for this transaction. - */ - #ifdef SQLITE_ENABLE_ATOMIC_WRITE - PgHdr *pPg; - assert( isOpen(pPager->jfd) - || pPager->journalMode==PAGER_JOURNALMODE_OFF - || pPager->journalMode==PAGER_JOURNALMODE_WAL - ); - if( !zMaster && isOpen(pPager->jfd) - && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbOrigSize - && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) - ){ - /* Update the db file change counter via the direct-write method. The - ** following call will modify the in-memory representation of page 1 - ** to include the updated change counter and then write page 1 - ** directly to the database file. Because of the atomic-write - ** property of the host file-system, this is safe. - */ - rc = pager_incr_changecounter(pPager, 1); - }else{ - rc = sqlite3JournalCreate(pPager->jfd); - if( rc==SQLITE_OK ){ - rc = pager_incr_changecounter(pPager, 0); - } - } - #else - rc = pager_incr_changecounter(pPager, 0); - #endif - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. This can only happen in auto-vacuum mode. - ** - ** Before reading the pages with page numbers larger than the - ** current value of Pager.dbSize, set dbSize back to the value - ** that it took at the start of the transaction. Otherwise, the - ** calls to sqlite3PagerGet() return zeroed pages instead of - ** reading data from the database file. - */ - #ifndef SQLITE_OMIT_AUTOVACUUM - if( pPager->dbSizedbOrigSize - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - ){ - Pgno i; /* Iterator variable */ - const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ - const Pgno dbSize = pPager->dbSize; /* Database image size */ - pPager->dbSize = pPager->dbOrigSize; - for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ - if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ - PgHdr *pPage; /* Page to journal */ - rc = sqlite3PagerGet(pPager, i, &pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - rc = sqlite3PagerWrite(pPage); - sqlite3PagerUnref(pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - } - pPager->dbSize = dbSize; - } - #endif - - /* Write the master journal name into the journal file. If a master - ** journal file name has already been written to the journal file, - ** or if zMaster is NULL (no master journal), then this call is a no-op. - */ - rc = writeMasterJournal(pPager, zMaster); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* Sync the journal file and write all dirty pages to the database. - ** If the atomic-update optimization is being used, this sync will not - ** create the journal file or perform any real IO. - ** - ** Because the change-counter page was just modified, unless the - ** atomic-update optimization is used it is almost certain that the - ** journal requires a sync here. However, in locking_mode=exclusive - ** on a system under memory pressure it is just possible that this is - ** not the case. In this case it is likely enough that the redundant - ** xSync() call will be changed to a no-op by the OS anyhow. - */ - rc = syncJournal(pPager, 0); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); - if( rc!=SQLITE_OK ){ - assert( rc!=SQLITE_IOERR_BLOCKED ); - goto commit_phase_one_exit; - } - sqlite3PcacheCleanAll(pPager->pPCache); - - /* If the file on disk is not the same size as the database image, - ** then use pager_truncate to grow or shrink the file here. - */ - if( pPager->dbSize!=pPager->dbFileSize ){ - Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); - assert( pPager->eState==PAGER_WRITER_DBMOD ); - rc = pager_truncate(pPager, nNew); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - - /* Finally, sync the database file. */ - if( !pPager->noSync && !noSync ){ - rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); - } - IOTRACE(("DBSYNC %p\n", pPager)) + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. + */ + if( isOpen(pPager->jfd) ){ + pager_error(pPager, pagerSyncHotJournal(pPager)); } + pagerUnlockAndRollback(pPager); } + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); + IOTRACE(("CLOSE %p\n", pPager)) + sqlite3OsClose(pPager->jfd); + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); -commit_phase_one_exit: - if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ - pPager->eState = PAGER_WRITER_FINISHED; - } - return rc; +#ifdef SQLITE_HAS_CODEC + if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); +#endif + + assert( !pPager->aSavepoint && !pPager->pInJournal ); + assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + + sqlite3_free(pPager); + return SQLITE_OK; } +#if !defined(NDEBUG) || defined(SQLITE_TEST) +/* +** Return the page number for page pPg. +*/ +SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage *pPg){ + return pPg->pgno; +} +#endif /* -** When this function is called, the database file has been completely -** updated to reflect the changes made by the current transaction and -** synced to disk. The journal file still exists in the file-system -** though, and if a failure occurs at this point it will eventually -** be used as a hot-journal and the current transaction rolled back. +** Increment the reference count for page pPg. +*/ +SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ + sqlite3PcacheRef(pPg); +} + +/* +** Sync the journal. In other words, make sure all the pages that have +** been written to the journal have actually reached the surface of the +** disk and can be restored in the event of a hot-journal rollback. ** -** This function finalizes the journal file, either by deleting, -** truncating or partially zeroing it, so that it cannot be used -** for hot-journal rollback. Once this is done the transaction is -** irrevocably committed. +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the the file-system, as follows: ** -** If an error occurs, an IO error code is returned and the pager -** moves into the error state. Otherwise, SQLITE_OK is returned. +** * If the journal file is an in-memory journal file, no action need +** be taken. +** +** * Otherwise, if the device does not support the SAFE_APPEND property, +** then the nRec field of the most recently written journal header +** is updated to contain the number of journal records that have +** been written following it. If the pager is operating in full-sync +** mode, then the journal file is synced before this field is updated. +** +** * If the device does not support the SEQUENTIAL property, then +** journal file is synced. +** +** Or, in pseudo-code: +** +** if( NOT ){ +** if( NOT SAFE_APPEND ){ +** if( ) xSync(); +** +** } +** if( NOT SEQUENTIAL ) xSync(); +** } +** +** If successful, this routine clears the PGHDR_NEED_SYNC flag of every +** page currently held in memory before returning SQLITE_OK. If an IO +** error is encountered, then the IO error code is returned to the caller. */ -SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ - int rc = SQLITE_OK; /* Return code */ - - /* This routine should not be called if a prior error has occurred. - ** But if (due to a coding error elsewhere in the system) it does get - ** called, just return the same error code without doing anything. */ - if( NEVER(pPager->errCode) ) return pPager->errCode; +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ - assert( pPager->eState==PAGER_WRITER_LOCKED - || pPager->eState==PAGER_WRITER_FINISHED - || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD ); assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); - /* An optimization. If the database was not actually modified during - ** this transaction, the pager is running in exclusive-mode and is - ** using persistent journals, then this function is a no-op. - ** - ** The start of the journal file currently contains a single journal - ** header with the nRec field set to 0. If such a journal is used as - ** a hot-journal during hot-journal rollback, 0 changes will be made - ** to the database file. So there is no need to zero the journal - ** header. Since the pager is in exclusive mode, there is no need - ** to drop any locks either. - */ - if( pPager->eState==PAGER_WRITER_LOCKED - && pPager->exclusiveMode - && pPager->journalMode==PAGER_JOURNALMODE_PERSIST - ){ - assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); - pPager->eState = PAGER_READER; - return SQLITE_OK; + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ + assert( !pPager->tempFile ); + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + assert( isOpen(pPager->jfd) ); + + if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + /* This block deals with an obscure problem. If the last connection + ** that wrote to this database was operating in persistent-journal + ** mode, then the journal file may at this point actually be larger + ** than Pager.journalOff bytes. If the next thing in the journal + ** file happens to be a journal-header (written as part of the + ** previous connection's transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the + ** hot-journal rollback following recovery. It may roll back all + ** of this connections data, then proceed to rolling back the old, + ** out-of-date data that follows it. Database corruption. + ** + ** To work around this, if the journal file does appear to contain + ** a valid header following Pager.journalOff, then write a 0x00 + ** byte to the start of it to prevent it from being recognized. + ** + ** Variable iNextHdrOffset is set to the offset at which this + ** problematic header will occur, if it exists. aMagic is used + ** as a temporary buffer to inspect the first couple of bytes of + ** the potential journal header. + */ + i64 iNextHdrOffset; + u8 aMagic[8]; + u8 zHeader[sizeof(aJournalMagic)+4]; + + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); + + iNextHdrOffset = journalHdrOffset(pPager); + rc = sqlite3OsRead(pPager->jfd, aMagic, 8, iNextHdrOffset); + if( rc==SQLITE_OK && 0==memcmp(aMagic, aJournalMagic, 8) ){ + static const u8 zerobyte = 0; + rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, iNextHdrOffset); + } + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + return rc; + } + + /* Write the nRec value into the journal file header. If in + ** full-synchronous mode, sync the journal first. This ensures that + ** all data has really hit the disk before nRec is updated to mark + ** it as a candidate for rollback. + ** + ** This is not required if the persistent media supports the + ** SAFE_APPEND property. Because in this case it is not possible + ** for garbage data to be appended to the file, the nRec field + ** is populated with 0xFFFFFFFF when the journal header is written + ** and never needs to be updated. + */ + if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + if( rc!=SQLITE_OK ) return rc; + } + IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); + rc = sqlite3OsWrite( + pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr + ); + if( rc!=SQLITE_OK ) return rc; + } + if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + ); + if( rc!=SQLITE_OK ) return rc; + } + + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } + }else{ + pPager->journalHdr = pPager->journalOff; + } } - PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - rc = pager_end_transaction(pPager, pPager->setMaster); - return pager_error(pPager, rc); + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); + return SQLITE_OK; } /* -** If a write transaction is open, then all changes made within the -** transaction are reverted and the current write-transaction is closed. -** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR -** state if an error occurs. -** -** If the pager is already in PAGER_ERROR state when this function is called, -** it returns Pager.errCode immediately. No work is performed in this case. +** The argument is the first in a linked list of dirty pages connected +** by the PgHdr.pDirty pointer. This function writes each one of the +** in-memory pages in the list to the database file. The argument may +** be NULL, representing an empty list. In this case this function is +** a no-op. ** -** Otherwise, in rollback mode, this function performs two functions: +** The pager must hold at least a RESERVED lock when this function +** is called. Before writing anything to the database file, this lock +** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, +** SQLITE_BUSY is returned and no data is written to the database file. +** +** If the pager is a temp-file pager and the actual file-system file +** is not yet open, it is created and opened before any data is +** written out. ** -** 1) It rolls back the journal file, restoring all database file and -** in-memory cache pages to the state they were in when the transaction -** was opened, and +** Once the lock has been upgraded and, if necessary, the file opened, +** the pages are written out to the database file in list order. Writing +** a page is skipped if it meets either of the following criteria: ** -** 2) It finalizes the journal file, so that it is not used for hot -** rollback at any point in the future. +** * The page number is greater than Pager.dbSize, or +** * The PGHDR_DONT_WRITE flag is set on the page. ** -** Finalization of the journal file (task 2) is only performed if the -** rollback is successful. +** If writing out a page causes the database file to grow, Pager.dbFileSize +** is updated accordingly. If page 1 is written out, then the value cached +** in Pager.dbFileVers[] is updated to match the new value stored in +** the database file. ** -** In WAL mode, all cache-entries containing data modified within the -** current transaction are either expelled from the cache or reverted to -** their pre-transaction state by re-reading data from the database or -** WAL files. The WAL transaction is then closed. +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot +** be obtained, SQLITE_BUSY is returned. */ -SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ +static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ int rc = SQLITE_OK; /* Return code */ - PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); - /* PagerRollback() is a no-op if called in READER or OPEN state. If - ** the pager is already in the ERROR state, the rollback is not - ** attempted here. Instead, the error code is returned to the caller. + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ + assert( !pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); + + /* If the file is a temp-file has not yet been opened, open it now. It + ** is not possible for rc to be other than SQLITE_OK if this branch + ** is taken, as pager_wait_on_lock() is a no-op for temp-files. */ - assert( assert_pager_state(pPager) ); - if( pPager->eState==PAGER_ERROR ) return pPager->errCode; - if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + if( !isOpen(pPager->fd) ){ + assert( pPager->tempFile && rc==SQLITE_OK ); + rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); + } - if( pagerUseWal(pPager) ){ - int rc2; - rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); - rc2 = pager_end_transaction(pPager, pPager->setMaster); - if( rc==SQLITE_OK ) rc = rc2; - }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ - rc = pager_end_transaction(pPager, 0); - }else{ - rc = pager_playback(pPager, 0); + /* Before the first write, give the VFS a hint of what the final + ** file size will be. + */ + assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; } - assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); + while( rc==SQLITE_OK && pList ){ + Pgno pgno = pList->pgno; - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error persistent. - */ - return pager_error(pPager, rc); -} + /* If there are dirty pages in the page cache with page numbers greater + ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to + ** make the file smaller (presumably by auto-vacuum code). Do not write + ** any such pages to the file. + ** + ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag + ** set (set by sqlite3PagerDontWrite()). + */ + if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ + i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ + char *pData; /* Data to write */ -/* -** Return TRUE if the database file is opened read-only. Return FALSE -** if the database is (in theory) writable. -*/ -SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ - return pPager->readOnly; -} + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + if( pList->pgno==1 ) pager_write_changecounter(pList); -/* -** Return the number of references to the pager. -*/ -SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ - return sqlite3PcacheRefCount(pPager->pPCache); -} + /* Encode the database */ + CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); -/* -** Return the approximate number of bytes of memory currently -** used by the pager and its associated cache. -*/ -SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ - int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) - + 5*sizeof(void*); - return perPageSize*sqlite3PcachePagecount(pPager->pPCache) - + sqlite3MallocSize(pPager) - + pPager->pageSize; + /* Write out the page data. */ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + + /* If page 1 was just written, update Pager.dbFileVers to match + ** the value now stored in the database file. If writing this + ** page caused the database file to grow, update dbFileSize. + */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); + } + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + + /* Update any backup objects copying the contents of this pager. */ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); + + PAGERTRACE(("STORE %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pList))); + IOTRACE(("PGOUT %p %d\n", pPager, pgno)); + PAGER_INCR(sqlite3_pager_writedb_count); + PAGER_INCR(pPager->nWrite); + }else{ + PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); + } + pager_set_pagehash(pList); + pList = pList->pDirty; + } + + return rc; } /* -** Return the number of references to the specified page. +** Ensure that the sub-journal file is open. If it is already open, this +** function is a no-op. +** +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() +** fails. */ -SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage *pPage){ - return sqlite3PcachePageRefcount(pPage); +static int openSubJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !isOpen(pPager->sjfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ + sqlite3MemJournalOpen(pPager->sjfd); + }else{ + rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + } + } + return rc; } -#ifdef SQLITE_TEST /* -** This routine is used for testing and analysis only. -*/ -SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ - static int a[11]; - a[0] = sqlite3PcacheRefCount(pPager->pPCache); - a[1] = sqlite3PcachePagecount(pPager->pPCache); - a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); - a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; - a[4] = pPager->eState; - a[5] = pPager->errCode; - a[6] = pPager->nHit; - a[7] = pPager->nMiss; - a[8] = 0; /* Used to be pPager->nOvfl */ - a[9] = pPager->nRead; - a[10] = pPager->nWrite; - return a; -} -#endif - -/* -** Return true if this is an in-memory pager. -*/ -SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; -} - -/* -** Check that there are at least nSavepoint savepoints open. If there are -** currently less than nSavepoints open, then open one or more savepoints -** to make up the difference. If the number of savepoints is already -** equal to nSavepoint, then this function is a no-op. +** Append a record of the current state of page pPg to the sub-journal. +** It is the callers responsibility to use subjRequiresPage() to check +** that it is really required before calling this function. ** -** If a memory allocation fails, SQLITE_NOMEM is returned. If an error -** occurs while opening the sub-journal file, then an IO error code is -** returned. Otherwise, SQLITE_OK. +** If successful, set the bit corresponding to pPg->pgno in the bitvecs +** for all open savepoints before returning. +** +** This function returns SQLITE_OK if everything is successful, an IO +** error code if the attempt to write to the sub-journal fails, or +** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint +** bitvec. */ -SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ - int rc = SQLITE_OK; /* Return code */ - int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ - - assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( assert_pager_state(pPager) ); - - if( nSavepoint>nCurrent && pPager->useJournal ){ - int ii; /* Iterator variable */ - PagerSavepoint *aNew; /* New Pager.aSavepoint array */ +static int subjournalPage(PgHdr *pPg){ + int rc = SQLITE_OK; + Pager *pPager = pPg->pPager; + if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ - /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a - ** malloc failure occurs while populating it in the for(...) loop below. - */ - aNew = (PagerSavepoint *)sqlite3Realloc( - pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + /* Open the sub-journal, if it has not already been opened */ + assert( pPager->useJournal ); + assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); + assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); + assert( pagerUseWal(pPager) + || pageInJournal(pPg) + || pPg->pgno>pPager->dbOrigSize ); - if( !aNew ){ - return SQLITE_NOMEM; - } - memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); - pPager->aSavepoint = aNew; + rc = openSubJournal(pPager); - /* Populate the PagerSavepoint structures just allocated. */ - for(ii=nCurrent; iidbSize; - if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ - aNew[ii].iOffset = pPager->journalOff; - }else{ - aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); - } - aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); - if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM; - } - if( pagerUseWal(pPager) ){ - sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + /* If the sub-journal was opened successfully (or was already open), + ** write the journal record into the file. */ + if( rc==SQLITE_OK ){ + void *pData = pPg->pData; + i64 offset = pPager->nSubRec*(4+pPager->pageSize); + char *pData2; + + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); } - pPager->nSavepoint = ii+1; } - assert( pPager->nSavepoint==nSavepoint ); - assertTruncateConstraint(pPager); } - + if( rc==SQLITE_OK ){ + pPager->nSubRec++; + assert( pPager->nSavepoint>0 ); + rc = addToSavepointBitvecs(pPager, pPg->pgno); + } return rc; } /* -** This function is called to rollback or release (commit) a savepoint. -** The savepoint to release or rollback need not be the most recently -** created savepoint. -** -** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. -** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with -** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes -** that have occurred since the specified savepoint was created. -** -** The savepoint to rollback or release is identified by parameter -** iSavepoint. A value of 0 means to operate on the outermost savepoint -** (the first created). A value of (Pager.nSavepoint-1) means operate -** on the most recently created savepoint. If iSavepoint is greater than -** (Pager.nSavepoint-1), then this function is a no-op. -** -** If a negative value is passed to this function, then the current -** transaction is rolled back. This is different to calling -** sqlite3PagerRollback() because this function does not terminate -** the transaction or unlock the database, it just restores the -** contents of the database to its original state. +** This function is called by the pcache layer when it has reached some +** soft memory limit. The first argument is a pointer to a Pager object +** (cast as a void*). The pager is always 'purgeable' (not an in-memory +** database). The second argument is a reference to a page that is +** currently dirty but has no outstanding references. The page +** is always associated with the Pager object passed as the first +** argument. ** -** In any case, all savepoints with an index greater than iSavepoint -** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), -** then savepoint iSavepoint is also destroyed. +** The job of this function is to make pPg clean by writing its contents +** out to the database file, if possible. This may involve syncing the +** journal file. ** -** This function may return SQLITE_NOMEM if a memory allocation fails, -** or an IO error code if an IO error occurs while rolling back a -** savepoint. If no errors occur, SQLITE_OK is returned. -*/ -SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = pPager->errCode; /* Return code */ +** If successful, sqlite3PcacheMakeClean() is called on the page and +** SQLITE_OK returned. If an IO error occurs while trying to make the +** page clean, the IO error code is returned. If the page cannot be +** made clean for some other reason, but no error occurs, then SQLITE_OK +** is returned by sqlite3PcacheMakeClean() is not called. +*/ +static int pagerStress(void *p, PgHdr *pPg){ + Pager *pPager = (Pager *)p; + int rc = SQLITE_OK; - assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); - assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); + assert( pPg->pPager==pPager ); + assert( pPg->flags&PGHDR_DIRTY ); - if( rc==SQLITE_OK && iSavepointnSavepoint ){ - int ii; /* Iterator variable */ - int nNew; /* Number of remaining savepoints after this op. */ + /* The doNotSyncSpill flag is set during times when doing a sync of + ** journal (and adding a new header) is not allowed. This occurs + ** during calls to sqlite3PagerWrite() while trying to journal multiple + ** pages belonging to the same sector. + ** + ** The doNotSpill flag inhibits all cache spilling regardless of whether + ** or not a sync is required. This is set during a rollback. + ** + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementaton it + ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. + */ + if( NEVER(pPager->errCode) ) return SQLITE_OK; + if( pPager->doNotSpill ) return SQLITE_OK; + if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ + return SQLITE_OK; + } - /* Figure out how many savepoints will still be active after this - ** operation. Store this value in nNew. Then free resources associated - ** with any savepoints that are destroyed by this operation. - */ - nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); - for(ii=nNew; iinSavepoint; ii++){ - sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + pPg->pDirty = 0; + if( pagerUseWal(pPager) ){ + /* Write a single frame for this page to the log. */ + if( subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); } - pPager->nSavepoint = nNew; - - /* If this is a release of the outermost savepoint, truncate - ** the sub-journal to zero bytes in size. */ - if( op==SAVEPOINT_RELEASE ){ - if( nNew==0 && isOpen(pPager->sjfd) ){ - /* Only truncate if it is an in-memory sub-journal. */ - if( sqlite3IsMemJournal(pPager->sjfd) ){ - rc = sqlite3OsTruncate(pPager->sjfd, 0); - assert( rc==SQLITE_OK ); - } - pPager->nSubRec = 0; - } + if( rc==SQLITE_OK ){ + rc = pagerWalFrames(pPager, pPg, 0, 0, 0); } - /* Else this is a rollback operation, playback the specified savepoint. - ** If this is a temp-file, it is possible that the journal file has - ** not yet been opened. In this case there have been no changes to - ** the database file, so the playback operation can be skipped. + }else{ + + /* Sync the journal file if required. */ + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); + } + + /* If the page number of this page is larger than the current size of + ** the database image, it may need to be written to the sub-journal. + ** This is because the call to pager_write_pagelist() below will not + ** actually write data to the file in this case. + ** + ** Consider the following sequence of events: + ** + ** BEGIN; + ** + ** + ** SAVEPOINT sp; + ** + ** pagerStress(page X) + ** ROLLBACK TO sp; + ** + ** If (X>Y), then when pagerStress is called page X will not be written + ** out to the database file, but will be dropped from the cache. Then, + ** following the "ROLLBACK TO sp" statement, reading page X will read + ** data from the database file. This will be the copy of page X as it + ** was when the transaction started, not as it was when "SAVEPOINT sp" + ** was executed. + ** + ** The solution is to write the current data for page X into the + ** sub-journal file now (if it is not already there), so that it will + ** be restored to its current value when the "ROLLBACK TO sp" is + ** executed. */ - else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ - PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; - rc = pagerPlaybackSavepoint(pPager, pSavepoint); - assert(rc!=SQLITE_DONE); + if( NEVER( + rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) + ) ){ + rc = subjournalPage(pPg); + } + + /* Write the contents of the page out to the database file. */ + if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); + rc = pager_write_pagelist(pPager, pPg); } } - return rc; -} - -/* -** Return the full pathname of the database file. -*/ -SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager){ - return pPager->zFilename; -} - -/* -** Return the VFS structure for the pager. -*/ -SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ - return pPager->pVfs; -} - -/* -** Return the file handle for the database file associated -** with the pager. This might return NULL if the file has -** not yet been opened. -*/ -SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ - return pPager->fd; -} - -/* -** Return the full pathname of the journal file. -*/ -SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ - return pPager->zJournal; -} + /* Mark the page as clean. */ + if( rc==SQLITE_OK ){ + PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); + sqlite3PcacheMakeClean(pPg); + } -/* -** Return true if fsync() calls are disabled for this pager. Return FALSE -** if fsync()s are executed normally. -*/ -SQLITE_PRIVATE int sqlite3PagerNosync(Pager *pPager){ - return pPager->noSync; + return pager_error(pPager, rc); } -#ifdef SQLITE_HAS_CODEC -/* -** Set or retrieve the codec for this pager -*/ -SQLITE_PRIVATE void sqlite3PagerSetCodec( - Pager *pPager, - void *(*xCodec)(void*,void*,Pgno,int), - void (*xCodecSizeChng)(void*,int,int), - void (*xCodecFree)(void*), - void *pCodec -){ - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); - pPager->xCodec = pPager->memDb ? 0 : xCodec; - pPager->xCodecSizeChng = xCodecSizeChng; - pPager->xCodecFree = xCodecFree; - pPager->pCodec = pCodec; - pagerReportSize(pPager); -} -SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ - return pPager->pCodec; -} -#endif -#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Move the page pPg to location pgno in the file. +** Allocate and initialize a new Pager object and put a pointer to it +** in *ppPager. The pager should eventually be freed by passing it +** to sqlite3PagerClose(). ** -** There must be no references to the page previously located at -** pgno (which we call pPgOld) though that page is allowed to be -** in cache. If the page previously located at pgno is not already -** in the rollback journal, it is not put there by by this routine. +** The zFilename argument is the path to the database file to open. +** If zFilename is NULL then a randomly-named temporary file is created +** and used as the file to be cached. Temporary files are be deleted +** automatically when they are closed. If zFilename is ":memory:" then +** all information is held in cache. It is never written to disk. +** This can be used to implement an in-memory database. ** -** References to the page pPg remain valid. Updating any -** meta-data associated with pPg (i.e. data stored in the nExtra bytes -** allocated along with the page) is the responsibility of the caller. +** The nExtra parameter specifies the number of bytes of space allocated +** along with each page reference. This space is available to the user +** via the sqlite3PagerGetExtra() API. ** -** A transaction must be active when this routine is called. It used to be -** required that a statement transaction was not active, but this restriction -** has been removed (CREATE INDEX needs to move a page when a statement -** transaction is active). +** The flags argument is used to specify properties that affect the +** operation of the pager. It should be passed some bitwise combination +** of the PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK flags. ** -** If the fourth argument, isCommit, is non-zero, then this page is being -** moved as part of a database reorganization just before the transaction -** is being committed. In this case, it is guaranteed that the database page -** pPg refers to will not be written to again within this transaction. +** The vfsFlags parameter is a bitmask to pass to the flags parameter +** of the xOpen() method of the supplied VFS when opening files. ** -** This function may return SQLITE_NOMEM or an IO error code if an error -** occurs. Otherwise, it returns SQLITE_OK. +** If the pager object is allocated and the specified file opened +** successfully, SQLITE_OK is returned and *ppPager set to point to +** the new pager object. If an error occurs, *ppPager is set to NULL +** and error code returned. This function may return SQLITE_NOMEM +** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or +** various SQLITE_IO_XXX errors. */ -SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ - PgHdr *pPgOld; /* The page being overwritten. */ - Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */ - int rc; /* Return code */ - Pgno origPgno; /* The original page number */ - - assert( pPg->nRef>0 ); - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - ); - assert( assert_pager_state(pPager) ); - - /* In order to be able to rollback, an in-memory database must journal - ** the page we are moving from. - */ - if( MEMDB ){ - rc = sqlite3PagerWrite(pPg); - if( rc ) return rc; - } +SQLITE_PRIVATE int sqlite3PagerOpen( + sqlite3_vfs *pVfs, /* The virtual file system to use */ + Pager **ppPager, /* OUT: Return the Pager structure here */ + const char *zFilename, /* Name of the database file to open */ + int nExtra, /* Extra bytes append to each in-memory page */ + int flags, /* flags controlling this file */ + int vfsFlags, /* flags passed through to sqlite3_vfs.xOpen() */ + void (*xReinit)(DbPage*) /* Function to reinitialize pages */ +){ + u8 *pPtr; + Pager *pPager = 0; /* Pager object to allocate and return */ + int rc = SQLITE_OK; /* Return code */ + int tempFile = 0; /* True for temp files (incl. in-memory files) */ + int memDb = 0; /* True if this is an in-memory file */ + int readOnly = 0; /* True if this is a read-only file */ + int journalFileSize; /* Bytes to allocate for each journal fd */ + char *zPathname = 0; /* Full path to database file */ + int nPathname = 0; /* Number of bytes in zPathname */ + int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ + int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ + int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ + int nUri = 0; /* Number of bytes of URI args at *zUri */ - /* If the page being moved is dirty and has not been saved by the latest - ** savepoint, then save the current contents of the page into the - ** sub-journal now. This is required to handle the following scenario: - ** - ** BEGIN; - ** - ** SAVEPOINT one; - ** - ** ROLLBACK TO one; - ** - ** If page X were not written to the sub-journal here, it would not - ** be possible to restore its contents when the "ROLLBACK TO one" - ** statement were is processed. - ** - ** subjournalPage() may need to allocate space to store pPg->pgno into - ** one or more savepoint bitvecs. This is the reason this function - ** may return SQLITE_NOMEM. + /* Figure out how much space is required for each journal file-handle + ** (there are two of them, the main journal and the sub-journal). This + ** is the maximum space required for an in-memory journal file handle + ** and a regular journal file-handle. Note that a "regular journal-handle" + ** may be a wrapper capable of caching the first portion of the journal + ** file in memory to implement the atomic-write optimization (see + ** source file journal.c). */ - if( pPg->flags&PGHDR_DIRTY - && subjRequiresPage(pPg) - && SQLITE_OK!=(rc = subjournalPage(pPg)) - ){ - return rc; + if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); + }else{ + journalFileSize = ROUND8(sqlite3MemJournalSize()); } - PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", - PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); - IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) + /* Set the output variable to NULL in case an error occurs. */ + *ppPager = 0; - /* If the journal needs to be sync()ed before page pPg->pgno can - ** be written to, store pPg->pgno in local variable needSyncPgno. - ** - ** If the isCommit flag is set, there is no need to remember that - ** the journal needs to be sync()ed before database page pPg->pgno - ** can be written to. The caller has already promised not to write to it. - */ - if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ - needSyncPgno = pPg->pgno; - assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); - assert( pPg->flags&PGHDR_DIRTY ); +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + zFilename = 0; } +#endif - /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for - ** page pgno before the 'move' operation, it needs to be retained - ** for the page moved there. + /* Compute and store the full pathname in an allocated buffer pointed + ** to by zPathname, length nPathname. Or, if this is a temporary file, + ** leave both nPathname and zPathname set to 0. */ - pPg->flags &= ~PGHDR_NEED_SYNC; - pPgOld = pager_lookup(pPager, pgno); - assert( !pPgOld || pPgOld->nRef==1 ); - if( pPgOld ){ - pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ - /* Do not discard pages from an in-memory database since we might - ** need to rollback later. Just move the page out of the way. */ - sqlite3PcacheMove(pPgOld, pPager->dbSize+1); - }else{ - sqlite3PcacheDrop(pPgOld); + if( zFilename && zFilename[0] ){ + const char *z; + nPathname = pVfs->mxPathname+1; + zPathname = sqlite3Malloc(nPathname*2); + if( zPathname==0 ){ + return SQLITE_NOMEM; + } + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ + z += sqlite3Strlen30(z)+1; + z += sqlite3Strlen30(z)+1; + } + nUri = &z[1] - zUri; + if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ + /* This branch is taken when the journal path required by + ** the database being opened will be more than pVfs->mxPathname + ** bytes in length. This means the database cannot be opened, + ** as it will not be possible to open the journal file or even + ** check for a hot-journal before reading. + */ + rc = SQLITE_CANTOPEN_BKPT; + } + if( rc!=SQLITE_OK ){ + sqlite3_free(zPathname); + return rc; } } - origPgno = pPg->pgno; - sqlite3PcacheMove(pPg, pgno); - sqlite3PcacheMakeDirty(pPg); - - /* For an in-memory database, make sure the original page continues - ** to exist, in case the transaction needs to roll back. Use pPgOld - ** as the original page since it has already been allocated. - */ - if( MEMDB ){ - assert( pPgOld ); - sqlite3PcacheMove(pPgOld, origPgno); - sqlite3PagerUnref(pPgOld); + /* Allocate memory for the Pager structure, PCache object, the + ** three file descriptors, the database file name and the journal + ** file name. The layout in memory is as follows: + ** + ** Pager object (sizeof(Pager) bytes) + ** PCache object (sqlite3PcacheSize() bytes) + ** Database file handle (pVfs->szOsFile bytes) + ** Sub-journal file handle (journalFileSize bytes) + ** Main journal file handle (journalFileSize bytes) + ** Database file name (nPathname+1 bytes) + ** Journal file name (nPathname+8+1 bytes) + */ + pPtr = (u8 *)sqlite3MallocZero( + ROUND8(sizeof(*pPager)) + /* Pager structure */ + ROUND8(pcacheSize) + /* PCache object */ + ROUND8(pVfs->szOsFile) + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + nPathname + 1 + nUri + /* zFilename */ + nPathname + 8 + 1 /* zJournal */ +#ifndef SQLITE_OMIT_WAL + + nPathname + 4 + 1 /* zWal */ +#endif + ); + assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); + if( !pPtr ){ + sqlite3_free(zPathname); + return SQLITE_NOMEM; + } + pPager = (Pager*)(pPtr); + pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); + pPager->fd = (sqlite3_file*)(pPtr += ROUND8(pcacheSize)); + pPager->sjfd = (sqlite3_file*)(pPtr += ROUND8(pVfs->szOsFile)); + pPager->jfd = (sqlite3_file*)(pPtr += journalFileSize); + pPager->zFilename = (char*)(pPtr += journalFileSize); + assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + + /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ + if( zPathname ){ + assert( nPathname>0 ); + pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); + memcpy(pPager->zFilename, zPathname, nPathname); + memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); + memcpy(pPager->zJournal, zPathname, nPathname); + memcpy(&pPager->zJournal[nPathname], "-journal", 8); + sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); +#ifndef SQLITE_OMIT_WAL + pPager->zWal = &pPager->zJournal[nPathname+8+1]; + memcpy(pPager->zWal, zPathname, nPathname); + memcpy(&pPager->zWal[nPathname], "-wal", 4); + sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); +#endif + sqlite3_free(zPathname); } + pPager->pVfs = pVfs; + pPager->vfsFlags = vfsFlags; - if( needSyncPgno ){ - /* If needSyncPgno is non-zero, then the journal file needs to be - ** sync()ed before any data is written to database file page needSyncPgno. - ** Currently, no such page exists in the page-cache and the - ** "is journaled" bitvec flag has been set. This needs to be remedied by - ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC - ** flag. + /* Open the pager file. + */ + if( zFilename && zFilename[0] ){ + int fout = 0; /* VFS flags returned by xOpen() */ + rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); + readOnly = (fout&SQLITE_OPEN_READONLY); + + /* If the file was successfully opened for read/write access, + ** choose a default page size in case we have to create the + ** database file. The default page size is the maximum of: ** - ** If the attempt to load the page into the page-cache fails, (due - ** to a malloc() or IO failure), clear the bit in the pInJournal[] - ** array. Otherwise, if the page is loaded and written again in - ** this transaction, it may be written to the database file before - ** it is synced into the journal file. This way, it may end up in - ** the journal file twice, but that is not a problem. + ** + SQLITE_DEFAULT_PAGE_SIZE, + ** + The value returned by sqlite3OsSectorSize() + ** + The largest page size that can be written atomically. */ - PgHdr *pPgHdr; - rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); - if( rc!=SQLITE_OK ){ - if( needSyncPgno<=pPager->dbOrigSize ){ - assert( pPager->pTmpSpace!=0 ); - sqlite3BitvecClear(pPager->pInJournal, needSyncPgno, pPager->pTmpSpace); + if( rc==SQLITE_OK && !readOnly ){ + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } } - return rc; +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } + } + } +#endif } - pPgHdr->flags |= PGHDR_NEED_SYNC; - sqlite3PcacheMakeDirty(pPgHdr); - sqlite3PagerUnref(pPgHdr); + }else{ + /* If a temporary file is requested, it is not opened immediately. + ** In this case we accept the default page size and delay actually + ** opening the file until the first call to OsWrite(). + ** + ** This branch is also run for an in-memory database. An in-memory + ** database is the same as a temp-file that is never written out to + ** disk and uses an in-memory rollback journal. + */ + tempFile = 1; + pPager->eState = PAGER_READER; + pPager->eLock = EXCLUSIVE_LOCK; + readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } - return SQLITE_OK; -} -#endif + /* The following call to PagerSetPagesize() serves to set the value of + ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. + */ + if( rc==SQLITE_OK ){ + assert( pPager->memDb==0 ); + rc = sqlite3PagerSetPagesize(pPager, &szPageDflt, -1); + testcase( rc!=SQLITE_OK ); + } -/* -** Return a pointer to the data for the specified page. -*/ -SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *pPg){ - assert( pPg->nRef>0 || pPg->pPager->memDb ); - return pPg->pData; -} + /* If an error occurred in either of the blocks above, free the + ** Pager structure and close the file. + */ + if( rc!=SQLITE_OK ){ + assert( !pPager->pTmpSpace ); + sqlite3OsClose(pPager->fd); + sqlite3_free(pPager); + return rc; + } -/* -** Return a pointer to the Pager.nExtra bytes of "extra" space -** allocated along with the specified page. -*/ -SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *pPg){ - return pPg->pExtra; -} + /* Initialize the PCache object. */ + assert( nExtra<1000 ); + nExtra = ROUND8(nExtra); + sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); -/* -** Get/set the locking-mode for this pager. Parameter eMode must be one -** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or -** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then -** the locking-mode is set to the value specified. -** -** The returned value is either PAGER_LOCKINGMODE_NORMAL or -** PAGER_LOCKINGMODE_EXCLUSIVE, indicating the current (possibly updated) -** locking-mode. -*/ -SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ - assert( eMode==PAGER_LOCKINGMODE_QUERY - || eMode==PAGER_LOCKINGMODE_NORMAL - || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); - assert( PAGER_LOCKINGMODE_QUERY<0 ); - assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); - assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); - if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ - pPager->exclusiveMode = (u8)eMode; + PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); + IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) + + pPager->useJournal = (u8)useJournal; + pPager->noReadlock = (noReadlock && readOnly) ?1:0; + /* pPager->stmtOpen = 0; */ + /* pPager->stmtInUse = 0; */ + /* pPager->nRef = 0; */ + /* pPager->stmtSize = 0; */ + /* pPager->stmtJSize = 0; */ + /* pPager->nPage = 0; */ + pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; + /* pPager->state = PAGER_UNLOCK; */ +#if 0 + assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); +#endif + /* pPager->errMask = 0; */ + pPager->tempFile = (u8)tempFile; + assert( tempFile==PAGER_LOCKINGMODE_NORMAL + || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); + pPager->exclusiveMode = (u8)tempFile; + pPager->changeCountDone = pPager->tempFile; + pPager->memDb = (u8)memDb; + pPager->readOnly = (u8)readOnly; + assert( useJournal || pPager->tempFile ); + pPager->noSync = pPager->tempFile; + pPager->fullSync = pPager->noSync ?0:1; + pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = pPager->syncFlags; + /* pPager->pFirst = 0; */ + /* pPager->pFirstSynced = 0; */ + /* pPager->pLast = 0; */ + pPager->nExtra = (u16)nExtra; + pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; + assert( isOpen(pPager->fd) || tempFile ); + setSectorSize(pPager); + if( !useJournal ){ + pPager->journalMode = PAGER_JOURNALMODE_OFF; + }else if( memDb ){ + pPager->journalMode = PAGER_JOURNALMODE_MEMORY; } - return (int)pPager->exclusiveMode; + /* pPager->xBusyHandler = 0; */ + /* pPager->pBusyHandlerArg = 0; */ + pPager->xReiniter = xReinit; + /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + + *ppPager = pPager; + return SQLITE_OK; } + + /* -** Set the journal-mode for this pager. Parameter eMode must be one of: -** -** PAGER_JOURNALMODE_DELETE -** PAGER_JOURNALMODE_TRUNCATE -** PAGER_JOURNALMODE_PERSIST -** PAGER_JOURNALMODE_OFF -** PAGER_JOURNALMODE_MEMORY -** PAGER_JOURNALMODE_WAL +** This function is called after transitioning from PAGER_UNLOCK to +** PAGER_SHARED state. It tests if there is a hot journal present in +** the file-system for the given pager. A hot journal is one that +** needs to be played back. According to this function, a hot-journal +** file exists if the following criteria are met: ** -** The journalmode is set to the value specified if the change is allowed. -** The change may be disallowed for the following reasons: +** * The journal file exists in the file system, and +** * No process holds a RESERVED or greater lock on the database file, and +** * The database file itself is greater than 0 bytes in size, and +** * The first byte of the journal file exists and is not 0x00. ** -** * An in-memory database can only have its journal_mode set to _OFF -** or _MEMORY. +** If the current size of the database file is 0 but a journal file +** exists, that is probably an old journal left over from a prior +** database with the same name. In this case the journal file is +** just deleted using OsDelete, *pExists is set to 0 and SQLITE_OK +** is returned. ** -** * Temporary databases cannot have _WAL journalmode. +** This routine does not check if there is a master journal filename +** at the end of the file. If there is, and that master journal file +** does not exist, then the journal file is not really hot. In this +** case this routine will return a false-positive. The pager_playback() +** routine will discover that the journal file is not really hot and +** will not roll it back. ** -** The returned indicate the current (possibly updated) journal-mode. +** If a hot-journal file is found to exist, *pExists is set to 1 and +** SQLITE_OK returned. If no hot-journal file is present, *pExists is +** set to 0 and SQLITE_OK returned. If an IO error occurs while trying +** to determine whether or not a hot-journal file exists, the IO error +** code is returned and the value of *pExists is undefined. */ -SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ - u8 eOld = pPager->journalMode; /* Prior journalmode */ - -#ifdef SQLITE_DEBUG - /* The print_pager_state() routine is intended to be used by the debugger - ** only. We invoke it once here to suppress a compiler warning. */ - print_pager_state(pPager); -#endif - +static int hasHotJournal(Pager *pPager, int *pExists){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int rc = SQLITE_OK; /* Return code */ + int exists = 1; /* True if a journal file is present */ + int jrnlOpen = !!isOpen(pPager->jfd); - /* The eMode parameter is always valid */ - assert( eMode==PAGER_JOURNALMODE_DELETE - || eMode==PAGER_JOURNALMODE_TRUNCATE - || eMode==PAGER_JOURNALMODE_PERSIST - || eMode==PAGER_JOURNALMODE_OFF - || eMode==PAGER_JOURNALMODE_WAL - || eMode==PAGER_JOURNALMODE_MEMORY ); + assert( pPager->useJournal ); + assert( isOpen(pPager->fd) ); + assert( pPager->eState==PAGER_OPEN ); - /* This routine is only called from the OP_JournalMode opcode, and - ** the logic there will never allow a temporary file to be changed - ** to WAL mode. - */ - assert( pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL ); + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN + )); - /* Do allow the journalmode of an in-memory database to be set to - ** anything other than MEMORY or OFF - */ - if( MEMDB ){ - assert( eOld==PAGER_JOURNALMODE_MEMORY || eOld==PAGER_JOURNALMODE_OFF ); - if( eMode!=PAGER_JOURNALMODE_MEMORY && eMode!=PAGER_JOURNALMODE_OFF ){ - eMode = eOld; - } + *pExists = 0; + if( !jrnlOpen ){ + rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); } + if( rc==SQLITE_OK && exists ){ + int locked = 0; /* True if some process holds a RESERVED lock */ - if( eMode!=eOld ){ - - /* Change the journal mode. */ - assert( pPager->eState!=PAGER_ERROR ); - pPager->journalMode = (u8)eMode; - - /* When transistioning from TRUNCATE or PERSIST to any other journal - ** mode except WAL, unless the pager is in locking_mode=exclusive mode, - ** delete the journal file. + /* Race condition here: Another process might have been holding the + ** the RESERVED lock and have a journal open at the sqlite3OsAccess() + ** call above, but then delete the journal and drop the lock before + ** we get to the following sqlite3OsCheckReservedLock() call. If that + ** is the case, this routine might think there is a hot journal when + ** in fact there is none. This results in a false-positive which will + ** be dealt with by the playback routine. Ticket #3883. */ - assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); - assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); - assert( (PAGER_JOURNALMODE_DELETE & 5)==0 ); - assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 ); - assert( (PAGER_JOURNALMODE_OFF & 5)==0 ); - assert( (PAGER_JOURNALMODE_WAL & 5)==5 ); - - assert( isOpen(pPager->fd) || pPager->exclusiveMode ); - if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ + rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); + if( rc==SQLITE_OK && !locked ){ + Pgno nPage; /* Number of pages in database file */ - /* In this case we would like to delete the journal file. If it is - ** not possible, then that is not a problem. Deleting the journal file - ** here is an optimization only. - ** - ** Before deleting the journal file, obtain a RESERVED lock on the - ** database file. This ensures that the journal file is not deleted - ** while it is in use by some other client. + /* Check the size of the database file. If it consists of 0 pages, + ** then delete the journal file. See the header comment above for + ** the reasoning here. Delete the obsolete journal file under + ** a RESERVED lock to avoid race conditions and to avoid violating + ** [H33020]. */ - sqlite3OsClose(pPager->jfd); - if( pPager->eLock>=RESERVED_LOCK ){ - sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); - }else{ - int rc = SQLITE_OK; - int state = pPager->eState; - assert( state==PAGER_OPEN || state==PAGER_READER ); - if( state==PAGER_OPEN ){ - rc = sqlite3PagerSharedLock(pPager); - } - if( pPager->eState==PAGER_READER ){ - assert( rc==SQLITE_OK ); - rc = pagerLockDb(pPager, RESERVED_LOCK); - } - if( rc==SQLITE_OK ){ - sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); - } - if( rc==SQLITE_OK && state==PAGER_READER ){ - pagerUnlockDb(pPager, SHARED_LOCK); - }else if( state==PAGER_OPEN ){ - pager_unlock(pPager); + rc = pagerPagecount(pPager, &nPage); + if( rc==SQLITE_OK ){ + if( nPage==0 ){ + sqlite3BeginBenignMalloc(); + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } + sqlite3EndBenignMalloc(); + }else{ + /* The journal file exists and no other connection has a reserved + ** or greater lock on the database file. Now check that there is + ** at least one non-zero bytes at the start of the journal file. + ** If there is, then we consider this journal to be hot. If not, + ** it can be ignored. + */ + if( !jrnlOpen ){ + int f = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &f); + } + if( rc==SQLITE_OK ){ + u8 first = 0; + rc = sqlite3OsRead(pPager->jfd, (void *)&first, 1, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + if( !jrnlOpen ){ + sqlite3OsClose(pPager->jfd); + } + *pExists = (first!=0); + }else if( rc==SQLITE_CANTOPEN ){ + /* If we cannot open the rollback journal file in order to see if + ** its has a zero header, that might be due to an I/O error, or + ** it might be due to the race condition described above and in + ** ticket #3883. Either way, assume that the journal is hot. + ** This might be a false positive. But if it is, then the + ** automatic journal playback and recovery mechanism will deal + ** with it under an EXCLUSIVE lock where we do not need to + ** worry so much with race conditions. + */ + *pExists = 1; + rc = SQLITE_OK; + } } - assert( state==pPager->eState ); } } } - /* Return the new journal mode */ - return (int)pPager->journalMode; -} - -/* -** Return the current journal mode. -*/ -SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager *pPager){ - return (int)pPager->journalMode; -} - -/* -** Return TRUE if the pager is in a state where it is OK to change the -** journalmode. Journalmode changes can only happen when the database -** is unmodified. -*/ -SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ - assert( assert_pager_state(pPager) ); - if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; - if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; - return 1; + return rc; } /* -** Get/set the size-limit used for persistent journal files. +** This function is called to obtain a shared lock on the database file. +** It is illegal to call sqlite3PagerAcquire() until after this function +** has been successfully called. If a shared-lock is already held when +** this function is called, it is a no-op. ** -** Setting the size limit to -1 means no limit is enforced. -** An attempt to set a limit smaller than -1 is a no-op. +** The following operations are also performed by this function. +** +** 1) If the pager is currently in PAGER_OPEN state (no lock held +** on the database file), then an attempt is made to obtain a +** SHARED lock on the database file. Immediately after obtaining +** the SHARED lock, the file-system is checked for a hot-journal, +** which is played back if present. Following any hot-journal +** rollback, the contents of the cache are validated by checking +** the 'change-counter' field of the database file header and +** discarded if they are found to be invalid. +** +** 2) If the pager is running in exclusive-mode, and there are currently +** no outstanding references to any pages, and is in the error state, +** then an attempt is made to clear the error state by discarding +** the contents of the page cache and rolling back any open journal +** file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. */ -SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ - if( iLimit>=-1 ){ - pPager->journalSizeLimit = iLimit; - } - return pPager->journalSizeLimit; -} +SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ -/* -** Return a pointer to the pPager->pBackup variable. The backup module -** in backup.c maintains the content of this variable. This module -** uses it opaquely as an argument to sqlite3BackupRestart() and -** sqlite3BackupUpdate() only. -*/ -SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ - return &pPager->pBackup; -} + /* This routine is only called from b-tree and only when there are no + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. + */ + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } -#ifndef SQLITE_OMIT_WAL -/* -** This function is called when the user invokes "PRAGMA checkpoint". -*/ -SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager){ - int rc = SQLITE_OK; - if( pPager->pWal ){ - u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3WalCheckpoint(pPager->pWal, pPager->ckptSyncFlags, - pPager->pageSize, zBuf); - } - return rc; -} + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ -SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ - return sqlite3WalCallback(pPager->pWal); -} + assert( !MEMDB ); + assert( pPager->noReadlock==0 || pPager->readOnly ); -/* -** Return true if the underlying VFS for the given pager supports the -** primitives necessary for write-ahead logging. -*/ -SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ - const sqlite3_io_methods *pMethods = pPager->fd->pMethods; - return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); -} + if( pPager->noReadlock==0 ){ + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; + } + } -/* -** Attempt to take an exclusive lock on the database file. If a PENDING lock -** is obtained instead, immediately release it. -*/ -static int pagerExclusiveLock(Pager *pPager){ - int rc; /* Return code */ + /* If a journal file exists, and there is no RESERVED lock on the + ** database file, then it either needs to be played back or deleted. + */ + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); + } + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ + /* Get an EXCLUSIVE lock on the database file. At this point it is + ** important that a RESERVED lock is not obtained on the way to the + ** EXCLUSIVE lock. If it were, another process might open the + ** database file, detect the RESERVED lock, and conclude that the + ** database is safe to read while this process is still rolling the + ** hot-journal back. + ** + ** Because the intermediate RESERVED lock is not requested, any + ** other process attempting to access the database file will get to + ** this point in the code and fail to obtain its own EXCLUSIVE lock + ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. + */ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + goto failed; + } + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } + + /* Playback and delete the journal. Drop the database write + ** lock and reacquire the read lock. Purge the cache before + ** playing back the hot-journal so that we don't end up with + ** an inconsistent cache. Sync the hot journal before playing + ** it back since the process that crashed and left the hot journal + ** probably did not sync it and we are required to always sync + ** the journal before playing it back. + */ + if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); + rc = pagerSyncHotJournal(pPager); + if( rc==SQLITE_OK ){ + rc = pager_playback(pPager, 1); + pPager->eState = PAGER_OPEN; + } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); + } - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); - rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - /* If the attempt to grab the pending lock failed, release the - ** exclusive lock that may have been obtained instead. */ - pagerUnlockDb(pPager, SHARED_LOCK); - } + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } - return rc; -} + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) + ); + } -/* -** Call sqlite3WalOpen() to open the WAL handle. If the pager is in -** exclusive-locking mode when this function is called, take an EXCLUSIVE -** lock on the database file and use heap-memory to store the wal-index -** in. Otherwise, use the normal shared-memory. -*/ -static int pagerOpenWal(Pager *pPager){ - int rc = SQLITE_OK; + if( !pPager->tempFile + && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) + ){ + /* The shared-lock has just been acquired on the database file + ** and there are already pages in the cache (from a previous + ** read or write transaction). Check to see if the database + ** has been modified. If the database has changed, flush the + ** cache. + ** + ** Database changes is detected by looking at 15 bytes beginning + ** at offset 24 into the file. The first 4 of these 16 bytes are + ** a 32-bit counter that is incremented with each change. The + ** other bytes change randomly with each file change when + ** a codec is in use. + ** + ** There is a vanishingly small chance that a change will not be + ** detected. The chance of an undetected change is so small that + ** it can be neglected. + */ + Pgno nPage = 0; + char dbFileVers[sizeof(pPager->dbFileVers)]; - assert( pPager->pWal==0 && pPager->tempFile==0 ); - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock); + rc = pagerPagecount(pPager, &nPage); + if( rc ) goto failed; - /* If the pager is already in exclusive-mode, the WAL module will use - ** heap-memory for the wal-index instead of the VFS shared-memory - ** implementation. Take the exclusive lock now, before opening the WAL - ** file, to make sure this is safe. - */ - if( pPager->exclusiveMode ){ - rc = pagerExclusiveLock(pPager); + if( nPage>0 ){ + IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( rc!=SQLITE_OK ){ + goto failed; + } + }else{ + memset(dbFileVers, 0, sizeof(dbFileVers)); + } + + if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ + pager_reset(pPager); + } + } + + /* If there is a WAL file in the file-system, open this database in WAL + ** mode. Otherwise, the following function call is a no-op. + */ + rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL + assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif } - /* Open the connection to the log file. If this operation fails, - ** (e.g. due to malloc() failure), return an error code. - */ - if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal - ); + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); } + failed: + if( rc!=SQLITE_OK ){ + assert( !MEMDB ); + pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; + } return rc; } +/* +** If the reference count has reached zero, rollback any active +** transaction and unlock the pager. +** +** Except, in locking_mode=EXCLUSIVE when there is nothing to in +** the rollback journal, the unlock is not performed and there is +** nothing to rollback, so this routine is a no-op. +*/ +static void pagerUnlockIfUnused(Pager *pPager){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + pagerUnlockAndRollback(pPager); + } +} /* -** The caller must be holding a SHARED lock on the database file to call -** this function. +** Acquire a reference to page number pgno in pager pPager (a page +** reference has type DbPage*). If the requested reference is +** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. ** -** If the pager passed as the first argument is open on a real database -** file (not a temp file or an in-memory database), and the WAL file -** is not already open, make an attempt to open it now. If successful, -** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pbOpen is -** not modified in either case. +** If the requested page is already in the cache, it is returned. +** Otherwise, a new page object is allocated and populated with data +** read from the database file. In some cases, the pcache module may +** choose not to allocate a new page object and may reuse an existing +** object with no outstanding references. ** -** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK -** without doing anything. +** The extra data appended to a page is always initialized to zeros the +** first time a page is loaded into memory. If the page requested is +** already in the cache when this function is called, then the extra +** data is left as it was when the page object was last used. +** +** If the database image is smaller than the requested page or if a +** non-zero value is passed as the noContent parameter and the +** requested page is not already stored in the cache, then no +** actual disk read occurs. In this case the memory image of the +** page is initialized to all zeros. +** +** If noContent is true, it means that we do not care about the contents +** of the page. This occurs in two seperate scenarios: +** +** a) When reading a free-list leaf page from the database, and +** +** b) When a savepoint is being rolled back and we need to load +** a new page into the cache to be filled with the data read +** from the savepoint journal. +** +** If noContent is true, then the data returned is zeroed instead of +** being read from the database. Additionally, the bits corresponding +** to pgno in Pager.pInJournal (bitvec of pages already written to the +** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open +** savepoints are set. This means if the page is made writable at any +** point in the future, using a call to sqlite3PagerWrite(), its contents +** will not be journaled. This saves IO. +** +** The acquisition might fail for several reasons. In all cases, +** an appropriate error code is returned and *ppPage is set to NULL. +** +** See also sqlite3PagerLookup(). Both this routine and Lookup() attempt +** to find a page in the in-memory cache first. If the page is not already +** in memory, this routine goes to disk to read it in whereas Lookup() +** just returns 0. This routine acquires a read-lock the first time it +** has to go to disk, and could also playback an old journal if necessary. +** Since Lookup() never goes to disk, it never has to deal with locks +** or journal files. */ -SQLITE_PRIVATE int sqlite3PagerOpenWal( - Pager *pPager, /* Pager object */ - int *pbOpen /* OUT: Set to true if call is a no-op */ +SQLITE_PRIVATE int sqlite3PagerAcquire( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int noContent /* Do not bother reading content from disk if true */ ){ - int rc = SQLITE_OK; /* Return code */ + int rc; + PgHdr *pPg; + assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( pPager->eState==PAGER_OPEN || pbOpen ); - assert( pPager->eState==PAGER_READER || !pbOpen ); - assert( pbOpen==0 || *pbOpen==0 ); - assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - - if( !pPager->tempFile && !pPager->pWal ){ - if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; - /* Close any rollback journal previously open */ - sqlite3OsClose(pPager->jfd); + if( pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } - rc = pagerOpenWal(pPager); - if( rc==SQLITE_OK ){ - pPager->journalMode = PAGER_JOURNALMODE_WAL; - pPager->eState = PAGER_OPEN; - } + /* If the pager is in the error state, return an error immediately. + ** Otherwise, request the page from the PCache layer. */ + if( pPager->errCode!=SQLITE_OK ){ + rc = pPager->errCode; }else{ - *pbOpen = 1; + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } - return rc; -} + if( rc!=SQLITE_OK ){ + /* Either the call to sqlite3PcacheFetch() returned an error or the + ** pager was already in the error-state when this function was called. + ** Set pPg to 0 and jump to the exception handler. */ + pPg = 0; + goto pager_acquire_err; + } + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); -/* -** This function is called to close the connection to the log file prior -** to switching from WAL to rollback mode. -** -** Before closing the log file, this function attempts to take an -** EXCLUSIVE lock on the database file. If this cannot be obtained, an -** error (SQLITE_BUSY) is returned and the log connection is not closed. -** If successful, the EXCLUSIVE lock is not released before returning. -*/ -SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ - int rc = SQLITE_OK; + if( (*ppPage)->pPager && !noContent ){ + /* In this case the pcache already contains an initialized copy of + ** the page. Return without further ado. */ + assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); + PAGER_INCR(pPager->nHit); + return SQLITE_OK; - assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + }else{ + /* The pager cache has created a new page. Its content needs to + ** be initialized. */ - /* If the log file is not already open, but does exist in the file-system, - ** it may need to be checkpointed before the connection can switch to - ** rollback mode. Open it now so this can happen. - */ - if( !pPager->pWal ){ - int logexists = 0; - rc = pagerLockDb(pPager, SHARED_LOCK); - if( rc==SQLITE_OK ){ - rc = sqlite3OsAccess( - pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists - ); + PAGER_INCR(pPager->nMiss); + pPg = *ppPage; + pPg->pPager = pPager; + + /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page + ** number greater than this, or the unused locking-page, is requested. */ + if( pgno>PAGER_MAX_PGNO || pgno==PAGER_MJ_PGNO(pPager) ){ + rc = SQLITE_CORRUPT_BKPT; + goto pager_acquire_err; } - if( rc==SQLITE_OK && logexists ){ - rc = pagerOpenWal(pPager); + + if( MEMDB || pPager->dbSizefd) ){ + if( pgno>pPager->mxPgno ){ + rc = SQLITE_FULL; + goto pager_acquire_err; + } + if( noContent ){ + /* Failure to set the bits in the InJournal bit-vectors is benign. + ** It merely means that we might do some extra work to journal a + ** page that does not need to be journaled. Nevertheless, be sure + ** to test the case where a malloc error occurs while trying to set + ** a bit in a bit vector. + */ + sqlite3BeginBenignMalloc(); + if( pgno<=pPager->dbOrigSize ){ + TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pgno); + testcase( rc==SQLITE_NOMEM ); + } + TESTONLY( rc = ) addToSavepointBitvecs(pPager, pgno); + testcase( rc==SQLITE_NOMEM ); + sqlite3EndBenignMalloc(); + } + memset(pPg->pData, 0, pPager->pageSize); + IOTRACE(("ZERO %p %d\n", pPager, pgno)); + }else{ + assert( pPg->pPager==pPager ); + rc = readDbPage(pPg); + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } + pager_set_pagehash(pPg); } - - /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on - ** the database file, the log and log-summary files will be deleted. - */ - if( rc==SQLITE_OK && pPager->pWal ){ - rc = pagerExclusiveLock(pPager); - if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, - pPager->pageSize, (u8*)pPager->pTmpSpace); - pPager->pWal = 0; - } + + return SQLITE_OK; + +pager_acquire_err: + assert( rc!=SQLITE_OK ); + if( pPg ){ + sqlite3PcacheDrop(pPg); } + pagerUnlockIfUnused(pPager); + + *ppPage = 0; return rc; } -#ifdef SQLITE_HAS_CODEC /* -** This function is called by the wal module when writing page content -** into the log file. +** Acquire a page if it is already in the in-memory cache. Do +** not read the page from disk. Return a pointer to the page, +** or 0 if the page is not in cache. ** -** This function returns a pointer to a buffer containing the encrypted -** page content. If a malloc fails, this function may return NULL. +** See also sqlite3PagerGet(). The difference between this routine +** and sqlite3PagerGet() is that _get() will go to the disk and read +** in the page if the page is not already in cache. This routine +** returns NULL if the page is not in cache or if a disk I/O error +** has ever happened. */ -SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ - void *aData = 0; - CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); - return aData; +SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ + PgHdr *pPg = 0; + assert( pPager!=0 ); + assert( pgno!=0 ); + assert( pPager->pPCache!=0 ); + assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); + sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + return pPg; } -#endif /* SQLITE_HAS_CODEC */ - -#endif /* !SQLITE_OMIT_WAL */ - -#endif /* SQLITE_OMIT_DISKIO */ -/************** End of pager.c ***********************************************/ -/************** Begin file wal.c *********************************************/ /* -** 2010 February 1 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Release a page reference. ** -************************************************************************* +** If the number of references to the page drop to zero, then the +** page is added to the LRU list. When all references to all pages +** are released, a rollback occurs and the lock on the database is +** removed. +*/ +SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ + if( pPg ){ + Pager *pPager = pPg->pPager; + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); + } +} + +/* +** This function is called at the start of every write transaction. +** There must already be a RESERVED or EXCLUSIVE lock on the database +** file when this routine is called. ** -** This file contains the implementation of a write-ahead log (WAL) used in -** "journal_mode=WAL" mode. +** Open the journal file for pager pPager and write a journal header +** to the start of it. If there are active savepoints, open the sub-journal +** as well. This function is only used when the journal file is being +** opened to write a rollback log for a transaction. It is not used +** when opening a hot journal file to roll it back. ** -** WRITE-AHEAD LOG (WAL) FILE FORMAT +** If the journal file is already open (as it may be in exclusive mode), +** then this function just writes a journal header to the start of the +** already open file. ** -** A WAL file consists of a header followed by zero or more "frames". -** Each frame records the revised content of a single page from the -** database file. All changes to the database are recorded by writing -** frames into the WAL. Transactions commit when a frame is written that -** contains a commit marker. A single WAL can and usually does record -** multiple transactions. Periodically, the content of the WAL is -** transferred back into the database file in an operation called a -** "checkpoint". +** Whether or not the journal file is opened by this function, the +** Pager.pInJournal bitvec structure is allocated. ** -** A single WAL file can be used multiple times. In other words, the -** WAL can fill up with frames and then be checkpointed and then new -** frames can overwrite the old ones. A WAL always grows from beginning -** toward the end. Checksums and counters attached to each frame are -** used to determine which frames within the WAL are valid and which -** are leftovers from prior checkpoints. -** -** The WAL header is 32 bytes in size and consists of the following eight -** big-endian 32-bit unsigned integer values: -** -** 0: Magic number. 0x377f0682 or 0x377f0683 -** 4: File format version. Currently 3007000 -** 8: Database page size. Example: 1024 -** 12: Checkpoint sequence number -** 16: Salt-1, random integer incremented with each checkpoint -** 20: Salt-2, a different random integer changing with each ckpt -** 24: Checksum-1 (first part of checksum for first 24 bytes of header). -** 28: Checksum-2 (second part of checksum for first 24 bytes of header). -** -** Immediately following the wal-header are zero or more frames. Each -** frame consists of a 24-byte frame-header followed by a bytes -** of page data. The frame-header is six big-endian 32-bit unsigned -** integer values, as follows: -** -** 0: Page number. -** 4: For commit records, the size of the database image in pages -** after the commit. For all other records, zero. -** 8: Salt-1 (copied from the header) -** 12: Salt-2 (copied from the header) -** 16: Checksum-1. -** 20: Checksum-2. -** -** A frame is considered valid if and only if the following conditions are -** true: -** -** (1) The salt-1 and salt-2 values in the frame-header match -** salt values in the wal-header -** -** (2) The checksum values in the final 8 bytes of the frame-header -** exactly match the checksum computed consecutively on the -** WAL header and the first 8 bytes and the content of all frames -** up to and including the current frame. -** -** The checksum is computed using 32-bit big-endian integers if the -** magic number in the first 4 bytes of the WAL is 0x377f0683 and it -** is computed using little-endian if the magic number is 0x377f0682. -** The checksum values are always stored in the frame header in a -** big-endian format regardless of which byte order is used to compute -** the checksum. The checksum is computed by interpreting the input as -** an even number of unsigned 32-bit integers: x[0] through x[N]. The -** algorithm used for the checksum is as follows: -** -** for i from 0 to n-1 step 2: -** s0 += x[i] + s1; -** s1 += x[i+1] + s0; -** endfor -** -** Note that s0 and s1 are both weighted checksums using fibonacci weights -** in reverse order (the largest fibonacci weight occurs on the first element -** of the sequence being summed.) The s1 value spans all 32-bit -** terms of the sequence whereas s0 omits the final term. -** -** On a checkpoint, the WAL is first VFS.xSync-ed, then valid content of the -** WAL is transferred into the database, then the database is VFS.xSync-ed. -** The VFS.xSync operations serve as write barriers - all writes launched -** before the xSync must complete before any write that launches after the -** xSync begins. -** -** After each checkpoint, the salt-1 value is incremented and the salt-2 -** value is randomized. This prevents old and new frames in the WAL from -** being considered valid at the same time and being checkpointing together -** following a crash. -** -** READER ALGORITHM -** -** To read a page from the database (call it page number P), a reader -** first checks the WAL to see if it contains page P. If so, then the -** last valid instance of page P that is a followed by a commit frame -** or is a commit frame itself becomes the value read. If the WAL -** contains no copies of page P that are valid and which are a commit -** frame or are followed by a commit frame, then page P is read from -** the database file. -** -** To start a read transaction, the reader records the index of the last -** valid frame in the WAL. The reader uses this recorded "mxFrame" value -** for all subsequent read operations. New transactions can be appended -** to the WAL, but as long as the reader uses its original mxFrame value -** and ignores the newly appended content, it will see a consistent snapshot -** of the database from a single point in time. This technique allows -** multiple concurrent readers to view different versions of the database -** content simultaneously. -** -** The reader algorithm in the previous paragraphs works correctly, but -** because frames for page P can appear anywhere within the WAL, the -** reader has to scan the entire WAL looking for page P frames. If the -** WAL is large (multiple megabytes is typical) that scan can be slow, -** and read performance suffers. To overcome this problem, a separate -** data structure called the wal-index is maintained to expedite the -** search for frames of a particular page. -** -** WAL-INDEX FORMAT -** -** Conceptually, the wal-index is shared memory, though VFS implementations -** might choose to implement the wal-index using a mmapped file. Because -** the wal-index is shared memory, SQLite does not support journal_mode=WAL -** on a network filesystem. All users of the database must be able to -** share memory. -** -** The wal-index is transient. After a crash, the wal-index can (and should -** be) reconstructed from the original WAL file. In fact, the VFS is required -** to either truncate or zero the header of the wal-index when the last -** connection to it closes. Because the wal-index is transient, it can -** use an architecture-specific format; it does not have to be cross-platform. -** Hence, unlike the database and WAL file formats which store all values -** as big endian, the wal-index can store multi-byte values in the native -** byte order of the host computer. -** -** The purpose of the wal-index is to answer this question quickly: Given -** a page number P, return the index of the last frame for page P in the WAL, -** or return NULL if there are no frames for page P in the WAL. -** -** The wal-index consists of a header region, followed by an one or -** more index blocks. -** -** The wal-index header contains the total number of frames within the WAL -** in the the mxFrame field. -** -** Each index block except for the first contains information on -** HASHTABLE_NPAGE frames. The first index block contains information on -** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and -** HASHTABLE_NPAGE are selected so that together the wal-index header and -** first index block are the same size as all other index blocks in the -** wal-index. -** -** Each index block contains two sections, a page-mapping that contains the -** database page number associated with each wal frame, and a hash-table -** that allows readers to query an index block for a specific page number. -** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE -** for the first index block) 32-bit page numbers. The first entry in the -** first index-block contains the database page number corresponding to the -** first frame in the WAL file. The first entry in the second index block -** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in -** the log, and so on. -** -** The last index block in a wal-index usually contains less than the full -** complement of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE) page-numbers, -** depending on the contents of the WAL file. This does not change the -** allocated size of the page-mapping array - the page-mapping array merely -** contains unused entries. -** -** Even without using the hash table, the last frame for page P -** can be found by scanning the page-mapping sections of each index block -** starting with the last index block and moving toward the first, and -** within each index block, starting at the end and moving toward the -** beginning. The first entry that equals P corresponds to the frame -** holding the content for that page. -** -** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers. -** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the -** hash table for each page number in the mapping section, so the hash -** table is never more than half full. The expected number of collisions -** prior to finding a match is 1. Each entry of the hash table is an -** 1-based index of an entry in the mapping section of the same -** index block. Let K be the 1-based index of the largest entry in -** the mapping section. (For index blocks other than the last, K will -** always be exactly HASHTABLE_NPAGE (4096) and for the last index block -** K will be (mxFrame%HASHTABLE_NPAGE).) Unused slots of the hash table -** contain a value of 0. -** -** To look for page P in the hash table, first compute a hash iKey on -** P as follows: -** -** iKey = (P * 383) % HASHTABLE_NSLOT -** -** Then start scanning entries of the hash table, starting with iKey -** (wrapping around to the beginning when the end of the hash table is -** reached) until an unused hash slot is found. Let the first unused slot -** be at index iUnused. (iUnused might be less than iKey if there was -** wrap-around.) Because the hash table is never more than half full, -** the search is guaranteed to eventually hit an unused entry. Let -** iMax be the value between iKey and iUnused, closest to iUnused, -** where aHash[iMax]==P. If there is no iMax entry (if there exists -** no hash slot such that aHash[i]==p) then page P is not in the -** current index block. Otherwise the iMax-th mapping entry of the -** current index block corresponds to the last entry that references -** page P. -** -** A hash search begins with the last index block and moves toward the -** first index block, looking for entries corresponding to page P. On -** average, only two or three slots in each index block need to be -** examined in order to either find the last entry for page P, or to -** establish that no such entry exists in the block. Each index block -** holds over 4000 entries. So two or three index blocks are sufficient -** to cover a typical 10 megabyte WAL file, assuming 1K pages. 8 or 10 -** comparisons (on average) suffice to either locate a frame in the -** WAL or to establish that the frame does not exist in the WAL. This -** is much faster than scanning the entire 10MB WAL. -** -** Note that entries are added in order of increasing K. Hence, one -** reader might be using some value K0 and a second reader that started -** at a later time (after additional transactions were added to the WAL -** and to the wal-index) might be using a different value K1, where K1>K0. -** Both readers can use the same hash table and mapping section to get -** the correct result. There may be entries in the hash table with -** K>K0 but to the first reader, those entries will appear to be unused -** slots in the hash table and so the first reader will get an answer as -** if no values greater than K0 had ever been inserted into the hash table -** in the first place - which is what reader one wants. Meanwhile, the -** second reader using K1 will see additional values that were inserted -** later, which is exactly what reader two wants. -** -** When a rollback occurs, the value of K is decreased. Hash table entries -** that correspond to frames greater than the new K value are removed -** from the hash table at this point. +** Return SQLITE_OK if everything is successful. Otherwise, return +** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or +** an IO error code if opening or writing the journal file fails. */ -#ifndef SQLITE_OMIT_WAL +static int pager_open_journal(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + assert( pPager->pInJournal==0 ); + + /* If already in the error state, this function is a no-op. But on + ** the other hand, this routine is never called if we are already in + ** an error state. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; -/* -** Trace output macros -*/ -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3WalTrace = 0; -# define WALTRACE(X) if(sqlite3WalTrace) sqlite3DebugPrintf X -#else -# define WALTRACE(X) -#endif + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + ); + #else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); + #endif + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; + } + + return rc; +} /* -** The maximum (and only) versions of the wal and wal-index formats -** that may be interpreted by this version of SQLite. +** Begin a write-transaction on the specified pager object. If a +** write-transaction has already been opened, this function is a no-op. ** -** If a client begins recovering a WAL file and finds that (a) the checksum -** values in the wal-header are correct and (b) the version field is not -** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN. +** If the exFlag argument is false, then acquire at least a RESERVED +** lock on the database file. If exFlag is true, then acquire at least +** an EXCLUSIVE lock. If such a lock is already held, no locking +** functions need be called. ** -** Similarly, if a client successfully reads a wal-index header (i.e. the -** checksum test is successful) and finds that the version field is not -** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite -** returns SQLITE_CANTOPEN. +** If the subjInMemory argument is non-zero, then any sub-journal opened +** within this transaction will be opened as an in-memory file. This +** has no effect if the sub-journal is already opened (as it may be when +** running in exclusive mode) or if the transaction does not require a +** sub-journal. If the subjInMemory argument is zero, then any required +** sub-journal is implemented in-memory if pPager is an in-memory database, +** or using a temporary file otherwise. */ -#define WAL_MAX_VERSION 3007000 -#define WALINDEX_MAX_VERSION 3007000 +SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ + int rc = SQLITE_OK; -/* -** Indices of various locking bytes. WAL_NREADER is the number -** of available reader locks and should be at least 3. -*/ -#define WAL_WRITE_LOCK 0 -#define WAL_ALL_BUT_WRITE 1 -#define WAL_CKPT_LOCK 1 -#define WAL_RECOVER_LOCK 2 -#define WAL_READ_LOCK(I) (3+(I)) -#define WAL_NREADER (SQLITE_SHM_NLOCK-3) + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + if( ALWAYS(pPager->eState==PAGER_READER) ){ + assert( pPager->pInJournal==0 ); -/* Object declarations */ -typedef struct WalIndexHdr WalIndexHdr; -typedef struct WalIterator WalIterator; -typedef struct WalCkptInfo WalCkptInfo; + if( pagerUseWal(pPager) ){ + /* If the pager is configured to use locking_mode=exclusive, and an + ** exclusive lock on the database is not already held, obtain it now. + */ + if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + return rc; + } + sqlite3WalExclusiveMode(pPager->pWal, 1); + } + /* Grab the write lock on the log file. If successful, upgrade to + ** PAGER_RESERVED state. Otherwise, return an error code to the caller. + ** The busy-handler is not invoked if another connection already + ** holds the write-lock. If possible, the upper layer will call it. + */ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + }else{ + /* Obtain a RESERVED lock on the database file. If the exFlag parameter + ** is true, then immediately upgrade this to an EXCLUSIVE lock. The + ** busy-handler callback can be used when upgrading to the EXCLUSIVE + ** lock, but not when obtaining the RESERVED lock. + */ + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } -/* -** The following object holds a copy of the wal-index header content. -** -** The actual header in the wal-index consists of two copies of this -** object. -** -** The szPage value can be any power of 2 between 512 and 32768, inclusive. -** Or it can be 1 to represent a 65536-byte page. The latter case was -** added in 3.7.1 when support for 64K pages was added. -*/ -struct WalIndexHdr { - u32 iVersion; /* Wal-index version */ - u32 unused; /* Unused (padding) field */ - u32 iChange; /* Counter incremented each transaction */ - u8 isInit; /* 1 when initialized */ - u8 bigEndCksum; /* True if checksums in WAL are big-endian */ - u16 szPage; /* Database page size in bytes. 1==64K */ - u32 mxFrame; /* Index of last valid frame in the WAL */ - u32 nPage; /* Size of database in pages */ - u32 aFrameCksum[2]; /* Checksum of last frame in log */ - u32 aSalt[2]; /* Two salt values copied from WAL header */ - u32 aCksum[2]; /* Checksum over all prior fields */ -}; + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. + */ + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; + } -/* -** A copy of the following object occurs in the wal-index immediately -** following the second copy of the WalIndexHdr. This object stores -** information used by checkpoint. -** -** nBackfill is the number of frames in the WAL that have been written -** back into the database. (We call the act of moving content from WAL to -** database "backfilling".) The nBackfill number is never greater than -** WalIndexHdr.mxFrame. nBackfill can only be increased by threads -** holding the WAL_CKPT_LOCK lock (which includes a recovery thread). -** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from -** mxFrame back to zero when the WAL is reset. -** -** There is one entry in aReadMark[] for each reader lock. If a reader -** holds read-lock K, then the value in aReadMark[K] is no greater than -** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) -** for any aReadMark[] means that entry is unused. aReadMark[0] is -** a special case; its value is never used and it exists as a place-holder -** to avoid having to offset aReadMark[] indexs by one. Readers holding -** WAL_READ_LOCK(0) always ignore the entire WAL and read all content -** directly from the database. -** -** The value of aReadMark[K] may only be changed by a thread that -** is holding an exclusive lock on WAL_READ_LOCK(K). Thus, the value of -** aReadMark[K] cannot changed while there is a reader is using that mark -** since the reader will be holding a shared lock on WAL_READ_LOCK(K). -** -** The checkpointer may only transfer frames from WAL to database where -** the frame numbers are less than or equal to every aReadMark[] that is -** in use (that is, every aReadMark[j] for which there is a corresponding -** WAL_READ_LOCK(j)). New readers (usually) pick the aReadMark[] with the -** largest value and will increase an unused aReadMark[] to mxFrame if there -** is not already an aReadMark[] equal to mxFrame. The exception to the -** previous sentence is when nBackfill equals mxFrame (meaning that everything -** in the WAL has been backfilled into the database) then new readers -** will choose aReadMark[0] which has value 0 and hence such reader will -** get all their all content directly from the database file and ignore -** the WAL. -** -** Writers normally append new frames to the end of the WAL. However, -** if nBackfill equals mxFrame (meaning that all WAL content has been -** written back into the database) and if no readers are using the WAL -** (in other words, if there are no WAL_READ_LOCK(i) where i>0) then -** the writer will first "reset" the WAL back to the beginning and start -** writing new content beginning at frame 1. -** -** We assume that 32-bit loads are atomic and so no locks are needed in -** order to read from any aReadMark[] entries. -*/ -struct WalCkptInfo { - u32 nBackfill; /* Number of WAL frames backfilled into DB */ - u32 aReadMark[WAL_NREADER]; /* Reader marks */ -}; -#define READMARK_NOT_USED 0xffffffff + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + } + PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); + return rc; +} -/* A block of WALINDEX_LOCK_RESERVED bytes beginning at -** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems -** only support mandatory file-locks, we do not read or write data -** from the region of the file on which locks are applied. +/* +** Mark a single data page as writeable. The page is written into the +** main journal or sub-journal as required. If the page is written into +** one of the journals, the corresponding bit is set in the +** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs +** of any open savepoints as appropriate. */ -#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2 + sizeof(WalCkptInfo)) -#define WALINDEX_LOCK_RESERVED 16 -#define WALINDEX_HDR_SIZE (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED) +static int pager_write(PgHdr *pPg){ + void *pData = pPg->pData; + Pager *pPager = pPg->pPager; + int rc = SQLITE_OK; -/* Size of header before each frame in wal */ -#define WAL_FRAME_HDRSIZE 24 + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. + */ + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); -/* Size of write ahead log header, including checksum. */ -/* #define WAL_HDRSIZE 24 */ -#define WAL_HDRSIZE 32 + /* If an error has been previously detected, report the same error + ** again. This should not happen, but the check provides robustness. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; -/* WAL magic value. Either this value, or the same value with the least -** significant bit also set (WAL_MAGIC | 0x00000001) is stored in 32-bit -** big-endian format in the first 4 bytes of a WAL file. -** -** If the LSB is set, then the checksums for each frame within the WAL -** file are calculated by treating all data as an array of 32-bit -** big-endian words. Otherwise, they are calculated by interpreting -** all data as 32-bit little-endian words. -*/ -#define WAL_MAGIC 0x377f0682 + /* Higher-level routines never call this function if database is not + ** writable. But check anyway, just for robustness. */ + if( NEVER(pPager->readOnly) ) return SQLITE_PERM; -/* -** Return the offset of frame iFrame in the write-ahead log file, -** assuming a database page size of szPage bytes. The offset returned -** is to the start of the write-ahead log frame-header. -*/ -#define walFrameOffset(iFrame, szPage) ( \ - WAL_HDRSIZE + ((iFrame)-1)*(i64)((szPage)+WAL_FRAME_HDRSIZE) \ -) + CHECK_PAGE(pPg); -/* -** An open write-ahead log file is represented by an instance of the -** following object. -*/ -struct Wal { - sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ - sqlite3_file *pDbFd; /* File handle for the database file */ - sqlite3_file *pWalFd; /* File handle for WAL file */ - u32 iCallback; /* Value to pass to log callback (or 0) */ - int nWiData; /* Size of array apWiData */ - volatile u32 **apWiData; /* Pointer to wal-index content in memory */ - u32 szPage; /* Database page size */ - i16 readLock; /* Which read lock is being held. -1 for none */ - u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ - u8 writeLock; /* True if in a write transaction */ - u8 ckptLock; /* True if holding a checkpoint lock */ - u8 readOnly; /* True if the WAL file is open read-only */ - WalIndexHdr hdr; /* Wal-index header for current transaction */ - const char *zWalName; /* Name of WAL file */ - u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ -#ifdef SQLITE_DEBUG - u8 lockError; /* True if a locking error has occurred */ -#endif -}; + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); -/* -** Candidate values for Wal.exclusiveMode. -*/ -#define WAL_NORMAL_MODE 0 -#define WAL_EXCLUSIVE_MODE 1 -#define WAL_HEAPMEMORY_MODE 2 + /* Mark the page as dirty. If the page has already been written + ** to the journal then we can return right away. + */ + sqlite3PcacheMakeDirty(pPg); + if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ + assert( !pagerUseWal(pPager) ); + }else{ + + /* The transaction journal now exists and we have a RESERVED or an + ** EXCLUSIVE lock on the main database file. Write the current page to + ** the transaction journal if it is not there already. + */ + if( !pageInJournal(pPg) && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; -/* -** Each page of the wal-index mapping contains a hash-table made up of -** an array of HASHTABLE_NSLOT elements of the following type. -*/ -typedef u16 ht_slot; + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); -/* -** This structure is used to implement an iterator that loops through -** all frames in the WAL in database page order. Where two or more frames -** correspond to the same database page, the iterator visits only the -** frame most recently written to the WAL (in other words, the frame with -** the largest index). -** -** The internals of this structure are only accessed by: -** -** walIteratorInit() - Create a new iterator, -** walIteratorNext() - Step an iterator, -** walIteratorFree() - Free an iterator. -** -** This functionality is used by the checkpoint code (see walCheckpoint()). -*/ -struct WalIterator { - int iPrior; /* Last result returned from the iterator */ - int nSegment; /* Size of the aSegment[] array */ - struct WalSegment { - int iNext; /* Next slot in aIndex[] not yet returned */ - ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ - u32 *aPgno; /* Array of page numbers. */ - int nEntry; /* Max size of aPgno[] and aIndex[] arrays */ - int iZero; /* Frame number associated with aPgno[0] */ - } aSegment[1]; /* One for every 32KB page in the WAL */ -}; + assert( pPager->journalHdr<=pPager->journalOff ); + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); -/* -** Define the parameters of the hash tables in the wal-index file. There -** is a hash-table following every HASHTABLE_NPAGE page numbers in the -** wal-index. -** -** Changing any of these constants will alter the wal-index format and -** create incompatibilities. -*/ -#define HASHTABLE_NPAGE 4096 /* Must be power of 2 */ -#define HASHTABLE_HASH_1 383 /* Should be prime */ -#define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; -/* -** The block of page numbers associated with the first hash-table in a -** wal-index is smaller than usual. This is so that there is a complete -** hash-table on each aligned 32KB page of the wal-index. -*/ -#define HASHTABLE_NPAGE_ONE (HASHTABLE_NPAGE - (WALINDEX_HDR_SIZE/sizeof(u32))) + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; -/* The wal-index is divided into pages of WALINDEX_PGSZ bytes each. */ -#define WALINDEX_PGSZ ( \ - sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ -) + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + if( rc!=SQLITE_OK ){ + assert( rc==SQLITE_NOMEM ); + return rc; + } + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + } + } + + /* If the statement journal is open and the page is not in it, + ** then write the current page to the statement journal. Note that + ** the statement journal format differs from the standard journal format + ** in that it omits the checksums and the header. + */ + if( subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } + } + + /* Update the database size and return. + */ + if( pPager->dbSizepgno ){ + pPager->dbSize = pPg->pgno; + } + return rc; +} /* -** Obtain a pointer to the iPage'th page of the wal-index. The wal-index -** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are -** numbered from zero. +** Mark a data page as writeable. This routine must be called before +** making changes to a page. The caller must check the return value +** of this function and be careful not to change any page data unless +** this routine returns SQLITE_OK. ** -** If this call is successful, *ppPage is set to point to the wal-index -** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, -** then an SQLite error code is returned and *ppPage is set to 0. +** The difference between this function and pager_write() is that this +** function also deals with the special case where 2 or more pages +** fit on a single disk sector. In this case all co-resident pages +** must have been written to the journal file before returning. +** +** If an error occurs, SQLITE_NOMEM or an IO error code is returned +** as appropriate. Otherwise, SQLITE_OK. */ -static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ +SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ int rc = SQLITE_OK; - /* Enlarge the pWal->apWiData[] array if required */ - if( pWal->nWiData<=iPage ){ - int nByte = sizeof(u32*)*(iPage+1); - volatile u32 **apNew; - apNew = (volatile u32 **)sqlite3_realloc((void *)pWal->apWiData, nByte); - if( !apNew ){ - *ppPage = 0; - return SQLITE_NOMEM; - } - memset((void*)&apNew[pWal->nWiData], 0, - sizeof(u32*)*(iPage+1-pWal->nWiData)); - pWal->apWiData = apNew; - pWal->nWiData = iPage+1; - } + PgHdr *pPg = pDbPage; + Pager *pPager = pPg->pPager; + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - /* Request a pointer to the required page from the VFS */ - if( pWal->apWiData[iPage]==0 ){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + + if( nPagePerSector>1 ){ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + + /* Set the doNotSyncSpill flag to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( pPager->doNotSyncSpill==0 ); + pPager->doNotSyncSpill++; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; }else{ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); + nPage = nPagePerSector; } - } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); - *ppPage = pWal->apWiData[iPage]; - assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnref(pPage); + } + } + }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnref(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnref(pPage); + } + } + } + + assert( pPager->doNotSyncSpill==1 ); + pPager->doNotSyncSpill--; + }else{ + rc = pager_write(pDbPage); + } return rc; } /* -** Return a pointer to the WalCkptInfo structure in the wal-index. +** Return TRUE if the page given in the argument was previously passed +** to sqlite3PagerWrite(). In other words, return TRUE if it is ok +** to change the content of the page. */ -static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ - assert( pWal->nWiData>0 && pWal->apWiData[0] ); - return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); +#ifndef NDEBUG +SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ + return pPg->flags&PGHDR_DIRTY; } +#endif /* -** Return a pointer to the WalIndexHdr structure in the wal-index. +** A call to this routine tells the pager that it is not necessary to +** write the information on page pPg back to the disk, even though +** that page might be marked as dirty. This happens, for example, when +** the page has been added as a leaf of the freelist and so its +** content no longer matters. +** +** The overlying software layer calls this routine when all of the data +** on the given page is unused. The pager marks the page as clean so +** that it does not get written to disk. +** +** Tests show that this optimization can quadruple the speed of large +** DELETE operations. */ -static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ - assert( pWal->nWiData>0 && pWal->apWiData[0] ); - return (volatile WalIndexHdr*)pWal->apWiData[0]; +SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); + IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) + pPg->flags |= PGHDR_DONT_WRITE; + pager_set_pagehash(pPg); + } } /* -** The argument to this macro must be of type u32. On a little-endian -** architecture, it returns the u32 value that results from interpreting -** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by intepreting the 4 bytes -** of the input value as a little-endian integer. -*/ -#define BYTESWAP32(x) ( \ - (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ - + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ -) - -/* -** Generate or extend an 8 byte checksum based on the data in -** array aByte[] and the initial values of aIn[0] and aIn[1] (or -** initial values of 0 and 0 if aIn==NULL). +** This routine is called to increment the value of the database file +** change-counter, stored as a 4-byte big-endian integer starting at +** byte offset 24 of the pager file. The secondary change counter at +** 92 is also updated, as is the SQLite version number at offset 96. ** -** The checksum is written back into aOut[] before returning. +** But this only happens if the pPager->changeCountDone flag is false. +** To avoid excess churning of page 1, the update only happens once. +** See also the pager_write_changecounter() routine that does an +** unconditional update of the change counters. ** -** nByte must be a positive multiple of 8. +** If the isDirectMode flag is zero, then this is done by calling +** sqlite3PagerWrite() on page 1, then modifying the contents of the +** page data. In this case the file will be updated when the current +** transaction is committed. +** +** The isDirectMode flag may only be non-zero if the library was compiled +** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, +** if isDirect is non-zero, then the database file is updated directly +** by writing an updated version of page 1 using a call to the +** sqlite3OsWrite() function. */ -static void walChecksumBytes( - int nativeCksum, /* True for native byte-order, false for non-native */ - u8 *a, /* Content to be checksummed */ - int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ - const u32 *aIn, /* Initial checksum value input */ - u32 *aOut /* OUT: Final checksum value output */ -){ - u32 s1, s2; - u32 *aData = (u32 *)a; - u32 *aEnd = (u32 *)&a[nByte]; +static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ + int rc = SQLITE_OK; - if( aIn ){ - s1 = aIn[0]; - s2 = aIn[1]; - }else{ - s1 = s2 = 0; - } + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); - assert( nByte>=8 ); - assert( (nByte&0x00000007)==0 ); + /* Declare and initialize constant integer 'isDirect'. If the + ** atomic-write optimization is enabled in this build, then isDirect + ** is initialized to the value passed as the isDirectMode parameter + ** to this function. Otherwise, it is always set to zero. + ** + ** The idea is that if the atomic-write optimization is not + ** enabled at compile time, the compiler can omit the tests of + ** 'isDirect' below, as well as the block enclosed in the + ** "if( isDirect )" condition. + */ +#ifndef SQLITE_ENABLE_ATOMIC_WRITE +# define DIRECT_MODE 0 + assert( isDirectMode==0 ); + UNUSED_PARAMETER(isDirectMode); +#else +# define DIRECT_MODE isDirectMode +#endif - if( nativeCksum ){ - do { - s1 += *aData++ + s2; - s2 += *aData++ + s1; - }while( aDatachangeCountDone && pPager->dbSize>0 ){ + PgHdr *pPgHdr; /* Reference to page 1 */ - aOut[0] = s1; - aOut[1] = s2; -} + assert( !pPager->tempFile && isOpen(pPager->fd) ); -static void walShmBarrier(Wal *pWal){ - if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ - sqlite3OsShmBarrier(pWal->pDbFd); + /* Open page 1 of the file for writing. */ + rc = sqlite3PagerGet(pPager, 1, &pPgHdr); + assert( pPgHdr==0 || rc==SQLITE_OK ); + + /* If page one was fetched successfully, and this function is not + ** operating in direct-mode, make page 1 writable. When not in + ** direct mode, page 1 is always held in cache and hence the PagerGet() + ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. + */ + if( !DIRECT_MODE && ALWAYS(rc==SQLITE_OK) ){ + rc = sqlite3PagerWrite(pPgHdr); + } + + if( rc==SQLITE_OK ){ + /* Actually do the update of the change counter */ + pager_write_changecounter(pPgHdr); + + /* If running in direct mode, write the contents of page 1 to the file. */ + if( DIRECT_MODE ){ + const void *zBuf; + assert( pPager->dbFileSize>0 ); + CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + } + if( rc==SQLITE_OK ){ + pPager->changeCountDone = 1; + } + }else{ + pPager->changeCountDone = 1; + } + } + + /* Release the page reference. */ + sqlite3PagerUnref(pPgHdr); } + return rc; } /* -** Write the header information in pWal->hdr into the wal-index. +** Sync the database file to disk. This is a no-op for in-memory databases +** or pages with the Pager.noSync flag set. ** -** The checksum on pWal->hdr is updated before it is written. +** If successful, or if called on a pager for which it is a no-op, this +** function returns SQLITE_OK. Otherwise, an IO error code is returned. */ -static void walIndexWriteHdr(Wal *pWal){ - volatile WalIndexHdr *aHdr = walIndexHdr(pWal); - const int nCksum = offsetof(WalIndexHdr, aCksum); - - assert( pWal->writeLock ); - pWal->hdr.isInit = 1; - pWal->hdr.iVersion = WALINDEX_MAX_VERSION; - walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); - memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); - walShmBarrier(pWal); - memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){ + int rc = SQLITE_OK; + if( !pPager->noSync ){ + assert( !MEMDB ); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); + }else if( isOpen(pPager->fd) ){ + assert( !MEMDB ); + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC_OMITTED, (void *)&rc); + } + return rc; } /* -** This function encodes a single frame header and writes it to a buffer -** supplied by the caller. A frame-header is made up of a series of -** 4-byte big-endian integers, as follows: +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. ** -** 0: Page number. -** 4: For commit records, the size of the database image in pages -** after the commit. For all other records, zero. -** 8: Salt-1 (copied from the wal-header) -** 12: Salt-2 (copied from the wal-header) -** 16: Checksum-1. -** 20: Checksum-2. +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. */ -static void walEncodeFrame( - Wal *pWal, /* The write-ahead log */ - u32 iPage, /* Database page number for frame */ - u32 nTruncate, /* New db size (or 0 for non-commit frames) */ - u8 *aData, /* Pointer to page data */ - u8 *aFrame /* OUT: Write encoded frame here */ -){ - int nativeCksum; /* True for native byte-order checksums */ - u32 *aCksum = pWal->hdr.aFrameCksum; - assert( WAL_FRAME_HDRSIZE==24 ); - sqlite3Put4byte(&aFrame[0], iPage); - sqlite3Put4byte(&aFrame[4], nTruncate); - memcpy(&aFrame[8], pWal->hdr.aSalt, 8); - - nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); - walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); - - sqlite3Put4byte(&aFrame[16], aCksum[0]); - sqlite3Put4byte(&aFrame[20], aCksum[1]); +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + return rc; } /* -** Check to see if the frame with header in aFrame[] and content -** in aData[] is valid. If it is a valid frame, fill *piPage and -** *pnTruncate and return true. Return if the frame is not valid. +** Sync the database file for the pager pPager. zMaster points to the name +** of a master journal file that should be written into the individual +** journal file. zMaster may be NULL, which is interpreted as no master +** journal (a single database transaction). +** +** This routine ensures that: +** +** * The database file change-counter is updated, +** * the journal is synced (unless the atomic-write optimization is used), +** * all dirty pages are written to the database file, +** * the database file is truncated (if required), and +** * the database file synced. +** +** The only thing that remains to commit the transaction is to finalize +** (delete, truncate or zero the first part of) the journal file (or +** delete the master journal file if specified). +** +** Note that if zMaster==NULL, this does not overwrite a previous value +** passed to an sqlite3PagerCommitPhaseOne() call. +** +** If the final parameter - noSync - is true, then the database file itself +** is not synced. The caller must call sqlite3PagerSync() directly to +** sync the database file before calling CommitPhaseTwo() to delete the +** journal file in this case. */ -static int walDecodeFrame( - Wal *pWal, /* The write-ahead log */ - u32 *piPage, /* OUT: Database page number for frame */ - u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ - u8 *aData, /* Pointer to page data (for checksum) */ - u8 *aFrame /* Frame data */ +SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( + Pager *pPager, /* Pager object */ + const char *zMaster, /* If not NULL, the master journal name */ + int noSync /* True to omit the xSync on the db file */ ){ - int nativeCksum; /* True for native byte-order checksums */ - u32 *aCksum = pWal->hdr.aFrameCksum; - u32 pgno; /* Page number of the frame */ - assert( WAL_FRAME_HDRSIZE==24 ); - - /* A frame is only valid if the salt values in the frame-header - ** match the salt values in the wal-header. - */ - if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){ - return 0; - } + int rc = SQLITE_OK; /* Return code */ - /* A frame is only valid if the page number is creater than zero. - */ - pgno = sqlite3Get4byte(&aFrame[0]); - if( pgno==0 ){ - return 0; - } + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); - /* A frame is only valid if a checksum of the WAL header, - ** all prior frams, the first 16 bytes of this frame-header, - ** and the frame-data matches the checksum in the last 8 - ** bytes of this frame-header. - */ - nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); - walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); - if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) - || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) - ){ - /* Checksum failed. */ - return 0; - } + /* If a prior error occurred, report that error again. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; - /* If we reach this point, the frame is valid. Return the page number - ** and the new database size. - */ - *piPage = pgno; - *pnTruncate = sqlite3Get4byte(&aFrame[4]); - return 1; -} + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", + pPager->zFilename, zMaster, pPager->dbSize)); + /* If no database changes have been made, return early. */ + if( pPager->eStatepBackup); }else{ - static char zName[15]; - sqlite3_snprintf(sizeof(zName), zName, "READ-LOCK[%d]", - lockIdx-WAL_READ_LOCK(0)); - return zName; + if( pagerUseWal(pPager) ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + PgHdr *pPageOne = 0; + if( pList==0 ){ + /* Must have at least one page for the WAL commit flag. + ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ + rc = sqlite3PagerGet(pPager, 1, &pPageOne); + pList = pPageOne; + pList->pDirty = 0; + } + assert( rc==SQLITE_OK ); + if( ALWAYS(pList) ){ + rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, + (pPager->fullSync ? pPager->syncFlags : 0) + ); + } + sqlite3PagerUnref(pPageOne); + if( rc==SQLITE_OK ){ + sqlite3PcacheCleanAll(pPager->pPCache); + } + }else{ + /* The following block updates the change-counter. Exactly how it + ** does this depends on whether or not the atomic-update optimization + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: + ** + ** * The file-system supports the atomic-write property for + ** blocks of size page-size, and + ** * This commit is not part of a multi-file transaction, and + ** * Exactly one page has been modified and store in the journal file. + ** + ** If the optimization was not enabled at compile time, then the + ** pager_incr_changecounter() function is called to update the change + ** counter in 'indirect-mode'. If the optimization is compiled in but + ** is not applicable to this transaction, call sqlite3JournalCreate() + ** to make sure the journal file has actually been created, then call + ** pager_incr_changecounter() to update the change-counter in indirect + ** mode. + ** + ** Otherwise, if the optimization is both enabled and applicable, + ** then call pager_incr_changecounter() to update the change-counter + ** in 'direct' mode. In this case the journal file will never be + ** created for this transaction. + */ + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + PgHdr *pPg; + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + if( !zMaster && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbOrigSize + && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); + } + } + #else + rc = pager_incr_changecounter(pPager, 0); + #endif + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* If this transaction has made the database smaller, then all pages + ** being discarded by the truncation must be written to the journal + ** file. This can only happen in auto-vacuum mode. + ** + ** Before reading the pages with page numbers larger than the + ** current value of Pager.dbSize, set dbSize back to the value + ** that it took at the start of the transaction. Otherwise, the + ** calls to sqlite3PagerGet() return zeroed pages instead of + ** reading data from the database file. + */ + #ifndef SQLITE_OMIT_AUTOVACUUM + if( pPager->dbSizedbOrigSize + && pPager->journalMode!=PAGER_JOURNALMODE_OFF + ){ + Pgno i; /* Iterator variable */ + const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ + const Pgno dbSize = pPager->dbSize; /* Database image size */ + pPager->dbSize = pPager->dbOrigSize; + for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ + if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ + PgHdr *pPage; /* Page to journal */ + rc = sqlite3PagerGet(pPager, i, &pPage); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + rc = sqlite3PagerWrite(pPage); + sqlite3PagerUnref(pPage); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + } + pPager->dbSize = dbSize; + } + #endif + + /* Write the master journal name into the journal file. If a master + ** journal file name has already been written to the journal file, + ** or if zMaster is NULL (no master journal), then this call is a no-op. + */ + rc = writeMasterJournal(pPager, zMaster); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. + */ + rc = syncJournal(pPager, 0); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_IOERR_BLOCKED ); + goto commit_phase_one_exit; + } + sqlite3PcacheCleanAll(pPager->pPCache); + + /* If the file on disk is not the same size as the database image, + ** then use pager_truncate to grow or shrink the file here. + */ + if( pPager->dbSize!=pPager->dbFileSize ){ + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + rc = pager_truncate(pPager, nNew); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + + /* Finally, sync the database file. */ + if( !noSync ){ + rc = sqlite3PagerSync(pPager); + } + IOTRACE(("DBSYNC %p\n", pPager)) + } } -} -#endif /*defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ - -/* -** Set or release locks on the WAL. Locks are either shared or exclusive. -** A lock cannot be moved directly between shared and exclusive - it must go -** through the unlocked state first. -** -** In locking_mode=EXCLUSIVE, all of these routines become no-ops. -*/ -static int walLockShared(Wal *pWal, int lockIdx){ - int rc; - if( pWal->exclusiveMode ) return SQLITE_OK; - rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, - SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); - WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, - walLockName(lockIdx), rc ? "failed" : "ok")); - VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) - return rc; -} -static void walUnlockShared(Wal *pWal, int lockIdx){ - if( pWal->exclusiveMode ) return; - (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, - SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); - WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); -} -static int walLockExclusive(Wal *pWal, int lockIdx, int n){ - int rc; - if( pWal->exclusiveMode ) return SQLITE_OK; - rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, - SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); - WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, - walLockName(lockIdx), n, rc ? "failed" : "ok")); - VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) +commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } return rc; } -static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ - if( pWal->exclusiveMode ) return; - (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, - SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); - WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal, - walLockName(lockIdx), n)); -} + /* -** Compute a hash on a page number. The resulting hash value must land -** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances -** the hash to the next value in the event of a collision. +** When this function is called, the database file has been completely +** updated to reflect the changes made by the current transaction and +** synced to disk. The journal file still exists in the file-system +** though, and if a failure occurs at this point it will eventually +** be used as a hot-journal and the current transaction rolled back. +** +** This function finalizes the journal file, either by deleting, +** truncating or partially zeroing it, so that it cannot be used +** for hot-journal rollback. Once this is done the transaction is +** irrevocably committed. +** +** If an error occurs, an IO error code is returned and the pager +** moves into the error state. Otherwise, SQLITE_OK is returned. */ -static int walHash(u32 iPage){ - assert( iPage>0 ); - assert( (HASHTABLE_NSLOT & (HASHTABLE_NSLOT-1))==0 ); - return (iPage*HASHTABLE_HASH_1) & (HASHTABLE_NSLOT-1); -} -static int walNextHash(int iPriorHash){ - return (iPriorHash+1)&(HASHTABLE_NSLOT-1); +SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine should not be called if a prior error has occurred. + ** But if (due to a coding error elsewhere in the system) it does get + ** called, just return the same error code without doing anything. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); + + /* An optimization. If the database was not actually modified during + ** this transaction, the pager is running in exclusive-mode and is + ** using persistent journals, then this function is a no-op. + ** + ** The start of the journal file currently contains a single journal + ** header with the nRec field set to 0. If such a journal is used as + ** a hot-journal during hot-journal rollback, 0 changes will be made + ** to the database file. So there is no need to zero the journal + ** header. Since the pager is in exclusive mode, there is no need + ** to drop any locks either. + */ + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode + && pPager->journalMode==PAGER_JOURNALMODE_PERSIST + ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; + return SQLITE_OK; + } + + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + rc = pager_end_transaction(pPager, pPager->setMaster); + return pager_error(pPager, rc); } -/* -** Return pointers to the hash table and page number array stored on -** page iHash of the wal-index. The wal-index is broken into 32KB pages -** numbered starting from 0. +/* +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. ** -** Set output variable *paHash to point to the start of the hash table -** in the wal-index file. Set *piZero to one less than the frame -** number of the first frame indexed by this hash table. If a -** slot in the hash table is set to N, it refers to frame number -** (*piZero+N) in the log. +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. ** -** Finally, set *paPgno so that *paPgno[1] is the page number of the -** first frame indexed by the hash table, frame (*piZero+1). +** Otherwise, in rollback mode, this function performs two functions: +** +** 1) It rolls back the journal file, restoring all database file and +** in-memory cache pages to the state they were in when the transaction +** was opened, and +** +** 2) It finalizes the journal file, so that it is not used for hot +** rollback at any point in the future. +** +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. +** +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. */ -static int walHashGet( - Wal *pWal, /* WAL handle */ - int iHash, /* Find the iHash'th table */ - volatile ht_slot **paHash, /* OUT: Pointer to hash index */ - volatile u32 **paPgno, /* OUT: Pointer to page number array */ - u32 *piZero /* OUT: Frame associated with *paPgno[0] */ -){ - int rc; /* Return code */ - volatile u32 *aPgno; - - rc = walIndexPage(pWal, iHash, &aPgno); - assert( rc==SQLITE_OK || iHash>0 ); +SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); - if( rc==SQLITE_OK ){ - u32 iZero; - volatile ht_slot *aHash; + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; - aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; - if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; - iZero = 0; - }else{ - iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + if( pagerUseWal(pPager) ){ + int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); + rc2 = pager_end_transaction(pPager, pPager->setMaster); + if( rc==SQLITE_OK ) rc = rc2; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + int eState = pPager->eState; + rc = pager_end_transaction(pPager, 0); + if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ + /* This can happen using journal_mode=off. Move the pager to the error + ** state to indicate that the contents of the cache may not be trusted. + ** Any active readers will get SQLITE_ABORT. + */ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + return rc; } - - *paPgno = &aPgno[-1]; - *paHash = aHash; - *piZero = iZero; + }else{ + rc = pager_playback(pPager, 0); } - return rc; + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); } /* -** Return the number of the wal-index page that contains the hash-table -** and page-number array that contain entries corresponding to WAL frame -** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages -** are numbered starting from 0. +** Return TRUE if the database file is opened read-only. Return FALSE +** if the database is (in theory) writable. */ -static int walFramePage(u32 iFrame){ - int iHash = (iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1) / HASHTABLE_NPAGE; - assert( (iHash==0 || iFrame>HASHTABLE_NPAGE_ONE) - && (iHash>=1 || iFrame<=HASHTABLE_NPAGE_ONE) - && (iHash<=1 || iFrame>(HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE)) - && (iHash>=2 || iFrame<=HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE) - && (iHash<=2 || iFrame>(HASHTABLE_NPAGE_ONE+2*HASHTABLE_NPAGE)) - ); - return iHash; +SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ + return pPager->readOnly; } /* -** Return the page number associated with frame iFrame in this WAL. +** Return the number of references to the pager. */ -static u32 walFramePgno(Wal *pWal, u32 iFrame){ - int iHash = walFramePage(iFrame); - if( iHash==0 ){ - return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; - } - return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ + return sqlite3PcacheRefCount(pPager->pPCache); } /* -** Remove entries from the hash table that point to WAL slots greater -** than pWal->hdr.mxFrame. -** -** This function is called whenever pWal->hdr.mxFrame is decreased due -** to a rollback or savepoint. -** -** At most only the hash table containing pWal->hdr.mxFrame needs to be -** updated. Any later hash tables will be automatically cleared when -** pWal->hdr.mxFrame advances to the point where those hash tables are -** actually needed. +** Return the approximate number of bytes of memory currently +** used by the pager and its associated cache. */ -static void walCleanupHash(Wal *pWal){ - volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ - volatile u32 *aPgno = 0; /* Page number array for hash table */ - u32 iZero = 0; /* frame == (aHash[x]+iZero) */ - int iLimit = 0; /* Zero values greater than this */ - int nByte; /* Number of bytes to zero in aPgno[] */ - int i; /* Used to iterate through aHash[] */ - - assert( pWal->writeLock ); - testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); - testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE ); - testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE+1 ); - - if( pWal->hdr.mxFrame==0 ) return; - - /* Obtain pointers to the hash-table and page-number array containing - ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed - ** that the page said hash-table and array reside on is already mapped. - */ - assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); - assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); - - /* Zero all hash-table entries that correspond to frame numbers greater - ** than pWal->hdr.mxFrame. - */ - iLimit = pWal->hdr.mxFrame - iZero; - assert( iLimit>0 ); - for(i=0; iiLimit ){ - aHash[i] = 0; - } - } - - /* Zero the entries in the aPgno array that correspond to frames with - ** frame numbers greater than pWal->hdr.mxFrame. - */ - nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); - memset((void *)&aPgno[iLimit+1], 0, nByte); - -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT - /* Verify that the every entry in the mapping region is still reachable - ** via the hash table even after the cleanup. - */ - if( iLimit ){ - int i; /* Loop counter */ - int iKey; /* Hash key */ - for(i=1; i<=iLimit; i++){ - for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==i ) break; - } - assert( aHash[iKey]==i ); - } - } -#endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ +SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ + int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) + + 5*sizeof(void*); + return perPageSize*sqlite3PcachePagecount(pPager->pPCache) + + sqlite3MallocSize(pPager) + + pPager->pageSize; } - /* -** Set an entry in the wal-index that will map database page number -** pPage into WAL frame iFrame. +** Return the number of references to the specified page. */ -static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ - int rc; /* Return code */ - u32 iZero = 0; /* One less than frame number of aPgno[1] */ - volatile u32 *aPgno = 0; /* Page number array */ - volatile ht_slot *aHash = 0; /* Hash table */ +SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage *pPage){ + return sqlite3PcachePageRefcount(pPage); +} - rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); +#ifdef SQLITE_TEST +/* +** This routine is used for testing and analysis only. +*/ +SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ + static int a[11]; + a[0] = sqlite3PcacheRefCount(pPager->pPCache); + a[1] = sqlite3PcachePagecount(pPager->pPCache); + a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; + a[5] = pPager->errCode; + a[6] = pPager->nHit; + a[7] = pPager->nMiss; + a[8] = 0; /* Used to be pPager->nOvfl */ + a[9] = pPager->nRead; + a[10] = pPager->nWrite; + return a; +} +#endif - /* Assuming the wal-index file was successfully mapped, populate the - ** page number array and hash table entry. - */ - if( rc==SQLITE_OK ){ - int iKey; /* Hash table key */ - int idx; /* Value to write to hash-table slot */ - int nCollide; /* Number of hash collisions */ +/* +** Return true if this is an in-memory pager. +*/ +SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ + return MEMDB; +} - idx = iFrame - iZero; - assert( idx <= HASHTABLE_NSLOT/2 + 1 ); - - /* If this is the first entry to be added to this hash-table, zero the - ** entire hash table and aPgno[] array before proceding. - */ - if( idx==1 ){ - int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); - memset((void*)&aPgno[1], 0, nByte); - } +/* +** Check that there are at least nSavepoint savepoints open. If there are +** currently less than nSavepoints open, then open one or more savepoints +** to make up the difference. If the number of savepoints is already +** equal to nSavepoint, then this function is a no-op. +** +** If a memory allocation fails, SQLITE_NOMEM is returned. If an error +** occurs while opening the sub-journal file, then an IO error code is +** returned. Otherwise, SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + int rc = SQLITE_OK; /* Return code */ + int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ - /* If the entry in aPgno[] is already set, then the previous writer - ** must have exited unexpectedly in the middle of a transaction (after - ** writing one or more dirty pages to the WAL to free up memory). - ** Remove the remnants of that writers uncommitted transaction from - ** the hash-table before writing any new entries. - */ - if( aPgno[idx] ){ - walCleanupHash(pWal); - assert( !aPgno[idx] ); - } + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); - /* Write the aPgno[] array entry and the hash-table slot. */ - nCollide = idx; - for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ - if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; - } - aPgno[idx] = iPage; - aHash[iKey] = (ht_slot)idx; + if( nSavepoint>nCurrent && pPager->useJournal ){ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT - /* Verify that the number of entries in the hash table exactly equals - ** the number of entries in the mapping region. + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. */ - { - int i; /* Loop counter */ - int nEntry = 0; /* Number of entries in the hash table */ - for(i=0; iaSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM; } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; - /* Verify that the every entry in the mapping region is reachable - ** via the hash table. This turns out to be a really, really expensive - ** thing to check, so only do this occasionally - not on every - ** iteration. - */ - if( (idx&0x3ff)==0 ){ - int i; /* Loop counter */ - for(i=1; i<=idx; i++){ - for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==i ) break; - } - assert( aHash[iKey]==i ); + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM; + } + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); } + pPager->nSavepoint = ii+1; } -#endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); } - return rc; } - /* -** Recover the wal-index by reading the write-ahead log file. +** This function is called to rollback or release (commit) a savepoint. +** The savepoint to release or rollback need not be the most recently +** created savepoint. ** -** This routine first tries to establish an exclusive lock on the -** wal-index to prevent other threads/processes from doing anything -** with the WAL or wal-index while recovery is running. The -** WAL_RECOVER_LOCK is also held so that other threads will know -** that this thread is running recovery. If unable to establish -** the necessary locks, this routine returns SQLITE_BUSY. -*/ -static int walIndexRecover(Wal *pWal){ - int rc; /* Return Code */ - i64 nSize; /* Size of log file */ - u32 aFrameCksum[2] = {0, 0}; - int iLock; /* Lock offset to lock for checkpoint */ - int nLock; /* Number of locks to hold */ - - /* Obtain an exclusive lock on all byte in the locking range not already - ** locked by the caller. The caller is guaranteed to have locked the - ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte. - ** If successful, the same bytes that are locked here are unlocked before - ** this function returns. - */ - assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); - assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); - assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); - assert( pWal->writeLock ); - iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; - nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); - if( rc ){ - return rc; - } - WALTRACE(("WAL%p: recovery begin...\n", pWal)); - - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - - rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); - if( rc!=SQLITE_OK ){ - goto recovery_error; - } +** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. +** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with +** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes +** that have occurred since the specified savepoint was created. +** +** The savepoint to rollback or release is identified by parameter +** iSavepoint. A value of 0 means to operate on the outermost savepoint +** (the first created). A value of (Pager.nSavepoint-1) means operate +** on the most recently created savepoint. If iSavepoint is greater than +** (Pager.nSavepoint-1), then this function is a no-op. +** +** If a negative value is passed to this function, then the current +** transaction is rolled back. This is different to calling +** sqlite3PagerRollback() because this function does not terminate +** the transaction or unlock the database, it just restores the +** contents of the database to its original state. +** +** In any case, all savepoints with an index greater than iSavepoint +** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), +** then savepoint iSavepoint is also destroyed. +** +** This function may return SQLITE_NOMEM if a memory allocation fails, +** or an IO error code if an IO error occurs while rolling back a +** savepoint. If no errors occur, SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ + int rc = pPager->errCode; /* Return code */ - if( nSize>WAL_HDRSIZE ){ - u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ - u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ - int szFrame; /* Number of bytes in buffer aFrame[] */ - u8 *aData; /* Pointer to data part of aFrame buffer */ - int iFrame; /* Index of last frame read */ - i64 iOffset; /* Next offset to read from log file */ - int szPage; /* Page size according to the log */ - u32 magic; /* Magic value read from WAL header */ - u32 version; /* Magic value read from WAL header */ + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); - /* Read in the WAL header. */ - rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); - if( rc!=SQLITE_OK ){ - goto recovery_error; - } + if( rc==SQLITE_OK && iSavepointnSavepoint ){ + int ii; /* Iterator variable */ + int nNew; /* Number of remaining savepoints after this op. */ - /* If the database page size is not a power of two, or is greater than - ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid - ** data. Similarly, if the 'magic' value is invalid, ignore the whole - ** WAL file. + /* Figure out how many savepoints will still be active after this + ** operation. Store this value in nNew. Then free resources associated + ** with any savepoints that are destroyed by this operation. */ - magic = sqlite3Get4byte(&aBuf[0]); - szPage = sqlite3Get4byte(&aBuf[8]); - if( (magic&0xFFFFFFFE)!=WAL_MAGIC - || szPage&(szPage-1) - || szPage>SQLITE_MAX_PAGE_SIZE - || szPage<512 - ){ - goto finished; - } - pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = szPage; - pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); - memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); - - /* Verify that the WAL header checksum is correct */ - walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, - aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum - ); - if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) - || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) - ){ - goto finished; - } - - /* Verify that the version number on the WAL format is one that - ** are able to understand */ - version = sqlite3Get4byte(&aBuf[4]); - if( version!=WAL_MAX_VERSION ){ - rc = SQLITE_CANTOPEN_BKPT; - goto finished; - } - - /* Malloc a buffer to read frames into. */ - szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc(szFrame); - if( !aFrame ){ - rc = SQLITE_NOMEM; - goto recovery_error; + nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); + for(ii=nNew; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - aData = &aFrame[WAL_FRAME_HDRSIZE]; - - /* Read all frames from the log file. */ - iFrame = 0; - for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ - u32 pgno; /* Database page number for frame */ - u32 nTruncate; /* dbsize field from frame header */ - int isValid; /* True if this frame is valid */ - - /* Read and decode the next log frame. */ - rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); - if( rc!=SQLITE_OK ) break; - isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); - if( !isValid ) break; - rc = walIndexAppend(pWal, ++iFrame, pgno); - if( rc!=SQLITE_OK ) break; + pPager->nSavepoint = nNew; - /* If nTruncate is non-zero, this is a commit record. */ - if( nTruncate ){ - pWal->hdr.mxFrame = iFrame; - pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; - aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + /* If this is a release of the outermost savepoint, truncate + ** the sub-journal to zero bytes in size. */ + if( op==SAVEPOINT_RELEASE ){ + if( nNew==0 && isOpen(pPager->sjfd) ){ + /* Only truncate if it is an in-memory sub-journal. */ + if( sqlite3IsMemJournal(pPager->sjfd) ){ + rc = sqlite3OsTruncate(pPager->sjfd, 0); + assert( rc==SQLITE_OK ); + } + pPager->nSubRec = 0; } } - - sqlite3_free(aFrame); - } - -finished: - if( rc==SQLITE_OK ){ - volatile WalCkptInfo *pInfo; - int i; - pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; - pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; - walIndexWriteHdr(pWal); - - /* Reset the checkpoint-header. This is safe because this thread is - ** currently holding locks that exclude all other readers, writers and - ** checkpointers. - */ - pInfo = walCkptInfo(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[0] = 0; - for(i=1; iaReadMark[i] = READMARK_NOT_USED; - - /* If more than one frame was recovered from the log file, report an - ** event via sqlite3_log(). This is to help with identifying performance - ** problems caused by applications routinely shutting down without - ** checkpointing the log file. + /* Else this is a rollback operation, playback the specified savepoint. + ** If this is a temp-file, it is possible that the journal file has + ** not yet been opened. In this case there have been no changes to + ** the database file, so the playback operation can be skipped. */ - if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName - ); + else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ + PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; + rc = pagerPlaybackSavepoint(pPager, pSavepoint); + assert(rc!=SQLITE_DONE); } } -recovery_error: - WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); - walUnlockExclusive(pWal, iLock, nLock); return rc; } /* -** Close an open wal-index. +** Return the full pathname of the database file. */ -static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - int i; - for(i=0; inWiData; i++){ - sqlite3_free((void *)pWal->apWiData[i]); - pWal->apWiData[i] = 0; - } - }else{ - sqlite3OsShmUnmap(pWal->pDbFd, isDelete); - } +SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager){ + return pPager->zFilename; } -/* -** Open a connection to the WAL file zWalName. The database file must -** already be opened on connection pDbFd. The buffer that zWalName points -** to must remain valid for the lifetime of the returned Wal* handle. -** -** A SHARED lock should be held on the database file when this function -** is called. The purpose of this SHARED lock is to prevent any other -** client from unlinking the WAL or wal-index file. If another process -** were to do this just after this client opened one of these files, the -** system would be badly broken. -** -** If the log file is successfully opened, SQLITE_OK is returned and -** *ppWal is set to point to a new WAL handle. If an error occurs, -** an SQLite error code is returned and *ppWal is left unmodified. +/* +** Return the VFS structure for the pager. */ -SQLITE_PRIVATE int sqlite3WalOpen( - sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ - sqlite3_file *pDbFd, /* The open database file */ - const char *zWalName, /* Name of the WAL file */ - int bNoShm, /* True to run in heap-memory mode */ - Wal **ppWal /* OUT: Allocated Wal handle */ -){ - int rc; /* Return Code */ - Wal *pRet; /* Object to allocate and return */ - int flags; /* Flags passed to OsOpen() */ - - assert( zWalName && zWalName[0] ); - assert( pDbFd ); - - /* In the amalgamation, the os_unix.c and os_win.c source files come before - ** this source file. Verify that the #defines of the locking byte offsets - ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. - */ -#ifdef WIN_SHM_BASE - assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); -#endif -#ifdef UNIX_SHM_BASE - assert( UNIX_SHM_BASE==WALINDEX_LOCK_OFFSET ); -#endif - - - /* Allocate an instance of struct Wal to return. */ - *ppWal = 0; - pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); - if( !pRet ){ - return SQLITE_NOMEM; - } - - pRet->pVfs = pVfs; - pRet->pWalFd = (sqlite3_file *)&pRet[1]; - pRet->pDbFd = pDbFd; - pRet->readLock = -1; - pRet->zWalName = zWalName; - pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); +SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ + return pPager->pVfs; +} - /* Open file handle on the write-ahead log file. */ - flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); - rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); - if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ - pRet->readOnly = 1; - } +/* +** Return the file handle for the database file associated +** with the pager. This might return NULL if the file has +** not yet been opened. +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ + return pPager->fd; +} - if( rc!=SQLITE_OK ){ - walIndexClose(pRet, 0); - sqlite3OsClose(pRet->pWalFd); - sqlite3_free(pRet); - }else{ - *ppWal = pRet; - WALTRACE(("WAL%d: opened\n", pRet)); - } - return rc; +/* +** Return the full pathname of the journal file. +*/ +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; } /* -** Find the smallest page number out of all pages held in the WAL that -** has not been returned by any prior invocation of this method on the -** same WalIterator object. Write into *piFrame the frame index where -** that page was last written into the WAL. Write into *piPage the page -** number. -** -** Return 0 on success. If there are no pages in the WAL with a page -** number larger than *piPage, then return 1. +** Return true if fsync() calls are disabled for this pager. Return FALSE +** if fsync()s are executed normally. */ -static int walIteratorNext( - WalIterator *p, /* Iterator */ - u32 *piPage, /* OUT: The page number of the next page */ - u32 *piFrame /* OUT: Wal frame index of next page */ -){ - u32 iMin; /* Result pgno must be greater than iMin */ - u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ - int i; /* For looping through segments */ - - iMin = p->iPrior; - assert( iMin<0xffffffff ); - for(i=p->nSegment-1; i>=0; i--){ - struct WalSegment *pSegment = &p->aSegment[i]; - while( pSegment->iNextnEntry ){ - u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; - if( iPg>iMin ){ - if( iPgiZero + pSegment->aIndex[pSegment->iNext]; - } - break; - } - pSegment->iNext++; - } - } - - *piPage = p->iPrior = iRet; - return (iRet==0xFFFFFFFF); +SQLITE_PRIVATE int sqlite3PagerNosync(Pager *pPager){ + return pPager->noSync; } +#ifdef SQLITE_HAS_CODEC /* -** This function merges two sorted lists into a single sorted list. +** Set or retrieve the codec for this pager */ -static void walMerge( - u32 *aContent, /* Pages in wal */ - ht_slot *aLeft, /* IN: Left hand input list */ - int nLeft, /* IN: Elements in array *paLeft */ - ht_slot **paRight, /* IN/OUT: Right hand input list */ - int *pnRight, /* IN/OUT: Elements in *paRight */ - ht_slot *aTmp /* Temporary buffer */ +SQLITE_PRIVATE void sqlite3PagerSetCodec( + Pager *pPager, + void *(*xCodec)(void*,void*,Pgno,int), + void (*xCodecSizeChng)(void*,int,int), + void (*xCodecFree)(void*), + void *pCodec ){ - int iLeft = 0; /* Current index in aLeft */ - int iRight = 0; /* Current index in aRight */ - int iOut = 0; /* Current index in output buffer */ - int nRight = *pnRight; - ht_slot *aRight = *paRight; - - assert( nLeft>0 && nRight>0 ); - while( iRightxCodecFree ) pPager->xCodecFree(pPager->pCodec); + pPager->xCodec = pPager->memDb ? 0 : xCodec; + pPager->xCodecSizeChng = xCodecSizeChng; + pPager->xCodecFree = xCodecFree; + pPager->pCodec = pCodec; + pagerReportSize(pPager); +} +SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ + return pPager->pCodec; +} +#endif - if( (iLeft=nRight || aContent[aLeft[iLeft]]pgno, if sync is required */ + int rc; /* Return code */ + Pgno origPgno; /* The original page number */ - aTmp[iOut++] = logpage; - if( iLeftnRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); - assert( iLeft>=nLeft || aContent[aLeft[iLeft]]>dbpage ); - assert( iRight>=nRight || aContent[aRight[iRight]]>dbpage ); + /* In order to be able to rollback, an in-memory database must journal + ** the page we are moving from. + */ + if( MEMDB ){ + rc = sqlite3PagerWrite(pPg); + if( rc ) return rc; } - *paRight = aLeft; - *pnRight = iOut; - memcpy(aLeft, aTmp, sizeof(aTmp[0])*iOut); -} - -/* -** Sort the elements in list aList, removing any duplicates. -*/ -static void walMergesort( - u32 *aContent, /* Pages in wal */ - ht_slot *aBuffer, /* Buffer of at least *pnList items to use */ - ht_slot *aList, /* IN/OUT: List to sort */ - int *pnList /* IN/OUT: Number of elements in aList[] */ -){ - struct Sublist { - int nList; /* Number of elements in aList */ - ht_slot *aList; /* Pointer to sub-list content */ - }; + /* If the page being moved is dirty and has not been saved by the latest + ** savepoint, then save the current contents of the page into the + ** sub-journal now. This is required to handle the following scenario: + ** + ** BEGIN; + ** + ** SAVEPOINT one; + ** + ** ROLLBACK TO one; + ** + ** If page X were not written to the sub-journal here, it would not + ** be possible to restore its contents when the "ROLLBACK TO one" + ** statement were is processed. + ** + ** subjournalPage() may need to allocate space to store pPg->pgno into + ** one or more savepoint bitvecs. This is the reason this function + ** may return SQLITE_NOMEM. + */ + if( pPg->flags&PGHDR_DIRTY + && subjRequiresPage(pPg) + && SQLITE_OK!=(rc = subjournalPage(pPg)) + ){ + return rc; + } - const int nList = *pnList; /* Size of input list */ - int nMerge = 0; /* Number of elements in list aMerge */ - ht_slot *aMerge = 0; /* List to be merged */ - int iList; /* Index into input list */ - int iSub = 0; /* Index into aSub array */ - struct Sublist aSub[13]; /* Array of sub-lists */ + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); + IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) - memset(aSub, 0, sizeof(aSub)); - assert( nList<=HASHTABLE_NPAGE && nList>0 ); - assert( HASHTABLE_NPAGE==(1<<(ArraySize(aSub)-1)) ); + /* If the journal needs to be sync()ed before page pPg->pgno can + ** be written to, store pPg->pgno in local variable needSyncPgno. + ** + ** If the isCommit flag is set, there is no need to remember that + ** the journal needs to be sync()ed before database page pPg->pgno + ** can be written to. The caller has already promised not to write to it. + */ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ + needSyncPgno = pPg->pgno; + assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPg->flags&PGHDR_DIRTY ); + } - for(iList=0; iListaList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); + /* If the cache contains a page with page-number pgno, remove it + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for + ** page pgno before the 'move' operation, it needs to be retained + ** for the page moved there. + */ + pPg->flags &= ~PGHDR_NEED_SYNC; + pPgOld = pager_lookup(pPager, pgno); + assert( !pPgOld || pPgOld->nRef==1 ); + if( pPgOld ){ + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); + if( MEMDB ){ + /* Do not discard pages from an in-memory database since we might + ** need to rollback later. Just move the page out of the way. */ + sqlite3PcacheMove(pPgOld, pPager->dbSize+1); + }else{ + sqlite3PcacheDrop(pPgOld); } - aSub[iSub].aList = aMerge; - aSub[iSub].nList = nMerge; } - for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); - } + origPgno = pPg->pgno; + sqlite3PcacheMove(pPg, pgno); + sqlite3PcacheMakeDirty(pPg); + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( MEMDB ){ + assert( pPgOld ); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnref(pPgOld); } - assert( aMerge==aList ); - *pnList = nMerge; -#ifdef SQLITE_DEBUG - { - int i; - for(i=1; i<*pnList; i++){ - assert( aContent[aList[i]] > aContent[aList[i-1]] ); + if( needSyncPgno ){ + /* If needSyncPgno is non-zero, then the journal file needs to be + ** sync()ed before any data is written to database file page needSyncPgno. + ** Currently, no such page exists in the page-cache and the + ** "is journaled" bitvec flag has been set. This needs to be remedied by + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC + ** flag. + ** + ** If the attempt to load the page into the page-cache fails, (due + ** to a malloc() or IO failure), clear the bit in the pInJournal[] + ** array. Otherwise, if the page is loaded and written again in + ** this transaction, it may be written to the database file before + ** it is synced into the journal file. This way, it may end up in + ** the journal file twice, but that is not a problem. + */ + PgHdr *pPgHdr; + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); + if( rc!=SQLITE_OK ){ + if( needSyncPgno<=pPager->dbOrigSize ){ + assert( pPager->pTmpSpace!=0 ); + sqlite3BitvecClear(pPager->pInJournal, needSyncPgno, pPager->pTmpSpace); + } + return rc; } + pPgHdr->flags |= PGHDR_NEED_SYNC; + sqlite3PcacheMakeDirty(pPgHdr); + sqlite3PagerUnref(pPgHdr); } + + return SQLITE_OK; +} #endif + +/* +** Return a pointer to the data for the specified page. +*/ +SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *pPg){ + assert( pPg->nRef>0 || pPg->pPager->memDb ); + return pPg->pData; } -/* -** Free an iterator allocated by walIteratorInit(). +/* +** Return a pointer to the Pager.nExtra bytes of "extra" space +** allocated along with the specified page. */ -static void walIteratorFree(WalIterator *p){ - sqlite3ScratchFree(p); +SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *pPg){ + return pPg->pExtra; } /* -** Construct a WalInterator object that can be used to loop over all -** pages in the WAL in ascending order. The caller must hold the checkpoint -** -** On success, make *pp point to the newly allocated WalInterator object -** return SQLITE_OK. Otherwise, return an error code. If this routine -** returns an error, the value of *pp is undefined. +** Get/set the locking-mode for this pager. Parameter eMode must be one +** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then +** the locking-mode is set to the value specified. ** -** The calling routine should invoke walIteratorFree() to destroy the -** WalIterator object when it has finished with it. +** The returned value is either PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE, indicating the current (possibly updated) +** locking-mode. */ -static int walIteratorInit(Wal *pWal, WalIterator **pp){ - WalIterator *p; /* Return value */ - int nSegment; /* Number of segments to merge */ - u32 iLast; /* Last frame in log */ - int nByte; /* Number of bytes to allocate */ - int i; /* Iterator variable */ - ht_slot *aTmp; /* Temp space used by merge-sort */ - int rc = SQLITE_OK; /* Return Code */ - - /* This routine only runs while holding the checkpoint lock. And - ** it only runs if there is actually content in the log (mxFrame>0). - */ - assert( pWal->ckptLock && pWal->hdr.mxFrame>0 ); - iLast = pWal->hdr.mxFrame; - - /* Allocate space for the WalIterator object. */ - nSegment = walFramePage(iLast) + 1; - nByte = sizeof(WalIterator) - + (nSegment-1)*sizeof(struct WalSegment) - + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3ScratchMalloc(nByte); - if( !p ){ - return SQLITE_NOMEM; - } - memset(p, 0, nByte); - p->nSegment = nSegment; - - /* Allocate temporary space used by the merge-sort routine. This block - ** of memory will be freed before this function returns. - */ - aTmp = (ht_slot *)sqlite3ScratchMalloc( - sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); - if( !aTmp ){ - rc = SQLITE_NOMEM; - } - - for(i=0; rc==SQLITE_OK && iaSegment[p->nSegment])[iZero]; - iZero++; - - for(j=0; jaSegment[i].iZero = iZero; - p->aSegment[i].nEntry = nEntry; - p->aSegment[i].aIndex = aIndex; - p->aSegment[i].aPgno = (u32 *)aPgno; - } - } - sqlite3ScratchFree(aTmp); - - if( rc!=SQLITE_OK ){ - walIteratorFree(p); +SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ + assert( eMode==PAGER_LOCKINGMODE_QUERY + || eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_QUERY<0 ); + assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); + assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); + if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ + pPager->exclusiveMode = (u8)eMode; } - *pp = p; - return rc; + return (int)pPager->exclusiveMode; } /* -** Copy as much content as we can from the WAL back into the database file -** in response to an sqlite3_wal_checkpoint() request or the equivalent. -** -** The amount of information copies from WAL to database might be limited -** by active readers. This routine will never overwrite a database page -** that a concurrent reader might be using. +** Set the journal-mode for this pager. Parameter eMode must be one of: ** -** All I/O barrier operations (a.k.a fsyncs) occur in this routine when -** SQLite is in WAL-mode in synchronous=NORMAL. That means that if -** checkpoints are always run by a background thread or background -** process, foreground threads will never block on a lengthy fsync call. +** PAGER_JOURNALMODE_DELETE +** PAGER_JOURNALMODE_TRUNCATE +** PAGER_JOURNALMODE_PERSIST +** PAGER_JOURNALMODE_OFF +** PAGER_JOURNALMODE_MEMORY +** PAGER_JOURNALMODE_WAL ** -** Fsync is called on the WAL before writing content out of the WAL and -** into the database. This ensures that if the new content is persistent -** in the WAL and can be recovered following a power-loss or hard reset. +** The journalmode is set to the value specified if the change is allowed. +** The change may be disallowed for the following reasons: ** -** Fsync is also called on the database file if (and only if) the entire -** WAL content is copied into the database file. This second fsync makes -** it safe to delete the WAL since the new content will persist in the -** database file. +** * An in-memory database can only have its journal_mode set to _OFF +** or _MEMORY. ** -** This routine uses and updates the nBackfill field of the wal-index header. -** This is the only routine tha will increase the value of nBackfill. -** (A WAL reset or recovery will revert nBackfill to zero, but not increase -** its value.) +** * Temporary databases cannot have _WAL journalmode. ** -** The caller must be holding sufficient locks to ensure that no other -** checkpoint is running (in any other thread or process) at the same -** time. +** The returned indicate the current (possibly updated) journal-mode. */ -static int walCheckpoint( - Wal *pWal, /* Wal connection */ - int sync_flags, /* Flags for OsSync() (or 0) */ - int nBuf, /* Size of zBuf in bytes */ - u8 *zBuf /* Temporary buffer to use */ -){ - int rc; /* Return code */ - int szPage; /* Database page-size */ - WalIterator *pIter = 0; /* Wal iterator context */ - u32 iDbpage = 0; /* Next database page to write */ - u32 iFrame = 0; /* Wal frame containing data for iDbpage */ - u32 mxSafeFrame; /* Max frame that can be backfilled */ - u32 mxPage; /* Max database page to write */ - int i; /* Loop counter */ - volatile WalCkptInfo *pInfo; /* The checkpoint status information */ +SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ + u8 eOld = pPager->journalMode; /* Prior journalmode */ - szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - if( pWal->hdr.mxFrame==0 ) return SQLITE_OK; +#ifdef SQLITE_DEBUG + /* The print_pager_state() routine is intended to be used by the debugger + ** only. We invoke it once here to suppress a compiler warning. */ + print_pager_state(pPager); +#endif - /* Allocate the iterator */ - rc = walIteratorInit(pWal, &pIter); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pIter ); - /*** TODO: Move this test out to the caller. Make it an assert() here ***/ - if( szPage!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - goto walcheckpoint_out; - } + /* The eMode parameter is always valid */ + assert( eMode==PAGER_JOURNALMODE_DELETE + || eMode==PAGER_JOURNALMODE_TRUNCATE + || eMode==PAGER_JOURNALMODE_PERSIST + || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_WAL + || eMode==PAGER_JOURNALMODE_MEMORY ); - /* Compute in mxSafeFrame the index of the last frame of the WAL that is - ** safe to write into the database. Frames beyond mxSafeFrame might - ** overwrite database pages that are in use by active readers and thus - ** cannot be backfilled from the WAL. + /* This routine is only called from the OP_JournalMode opcode, and + ** the logic there will never allow a temporary file to be changed + ** to WAL mode. */ - mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - pInfo = walCkptInfo(pWal); - for(i=1; iaReadMark[i]; - if( mxSafeFrame>=y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = READMARK_NOT_USED; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - }else{ - goto walcheckpoint_out; - } + assert( pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL ); + + /* Do allow the journalmode of an in-memory database to be set to + ** anything other than MEMORY or OFF + */ + if( MEMDB ){ + assert( eOld==PAGER_JOURNALMODE_MEMORY || eOld==PAGER_JOURNALMODE_OFF ); + if( eMode!=PAGER_JOURNALMODE_MEMORY && eMode!=PAGER_JOURNALMODE_OFF ){ + eMode = eOld; } } - if( pInfo->nBackfillnBackfill; + if( eMode!=eOld ){ - /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } + /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); + pPager->journalMode = (u8)eMode; - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. + /* When transistioning from TRUNCATE or PERSIST to any other journal + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, + ** delete the journal file. */ - if( rc==SQLITE_OK ){ - i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } - } + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)==0 ); + assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 ); + assert( (PAGER_JOURNALMODE_OFF & 5)==0 ); + assert( (PAGER_JOURNALMODE_WAL & 5)==5 ); - /* Iterate through the contents of the WAL, copying data to the db file. */ - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ - i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - iOffset = (iDbpage-1)*(i64)szPage; - testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - } + assert( isOpen(pPager->fd) || pPager->exclusiveMode ); + if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ - /* If work was actually accomplished... */ - if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + /* In this case we would like to delete the journal file. If it is + ** not possible, then that is not a problem. Deleting the journal file + ** here is an optimization only. + ** + ** Before deleting the journal file, obtain a RESERVED lock on the + ** database file. This ensures that the journal file is not deleted + ** while it is in use by some other client. + */ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); } - } - if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); } } - - /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); - }else if( rc==SQLITE_BUSY ){ - /* Reset the return code so as not to report a checkpoint failure - ** just because active readers prevent any backfill. - */ - rc = SQLITE_OK; } - walcheckpoint_out: - walIteratorFree(pIter); - return rc; + /* Return the new journal mode */ + return (int)pPager->journalMode; } /* -** Close a connection to a log file. +** Return the current journal mode. */ -SQLITE_PRIVATE int sqlite3WalClose( - Wal *pWal, /* Wal to close */ - int sync_flags, /* Flags to pass to OsSync() (or 0) */ - int nBuf, - u8 *zBuf /* Buffer of at least nBuf bytes */ -){ - int rc = SQLITE_OK; - if( pWal ){ - int isDelete = 0; /* True to unlink wal and wal-index files */ - - /* If an EXCLUSIVE lock can be obtained on the database file (using the - ** ordinary, rollback-mode locking methods, this guarantees that the - ** connection associated with this log file is the only connection to - ** the database. In this case checkpoint the database and unlink both - ** the wal and wal-index files. - ** - ** The EXCLUSIVE lock is not released before returning. - */ - rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); - if( rc==SQLITE_OK ){ - if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ - pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; - } - rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf); - if( rc==SQLITE_OK ){ - isDelete = 1; - } - } +SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager *pPager){ + return (int)pPager->journalMode; +} - walIndexClose(pWal, isDelete); - sqlite3OsClose(pWal->pWalFd); - if( isDelete ){ - sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); - } - WALTRACE(("WAL%p: closed\n", pWal)); - sqlite3_free((void *)pWal->apWiData); - sqlite3_free(pWal); - } - return rc; +/* +** Return TRUE if the pager is in a state where it is OK to change the +** journalmode. Journalmode changes can only happen when the database +** is unmodified. +*/ +SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; + if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; + return 1; } /* -** Try to read the wal-index header. Return 0 on success and 1 if -** there is a problem. -** -** The wal-index is in shared memory. Another thread or process might -** be writing the header at the same time this procedure is trying to -** read it, which might result in inconsistency. A dirty read is detected -** by verifying that both copies of the header are the same and also by -** a checksum on the header. -** -** If and only if the read is consistent and the header is different from -** pWal->hdr, then pWal->hdr is updated to the content of the new header -** and *pChanged is set to 1. +** Get/set the size-limit used for persistent journal files. ** -** If the checksum cannot be verified return non-zero. If the header -** is read successfully and the checksum verified, return zero. +** Setting the size limit to -1 means no limit is enforced. +** An attempt to set a limit smaller than -1 is a no-op. */ -static int walIndexTryHdr(Wal *pWal, int *pChanged){ - u32 aCksum[2]; /* Checksum on the header content */ - WalIndexHdr h1, h2; /* Two copies of the header content */ - WalIndexHdr volatile *aHdr; /* Header in shared memory */ - - /* The first page of the wal-index must be mapped at this point. */ - assert( pWal->nWiData>0 && pWal->apWiData[0] ); +SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ + if( iLimit>=-1 ){ + pPager->journalSizeLimit = iLimit; + sqlite3WalLimit(pPager->pWal, iLimit); + } + return pPager->journalSizeLimit; +} - /* Read the header. This might happen concurrently with a write to the - ** same area of shared memory on a different CPU in a SMP, - ** meaning it is possible that an inconsistent snapshot is read - ** from the file. If this happens, return non-zero. - ** - ** There are two copies of the header at the beginning of the wal-index. - ** When reading, read [0] first then [1]. Writes are in the reverse order. - ** Memory barriers are used to prevent the compiler or the hardware from - ** reordering the reads and writes. - */ - aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); - walShmBarrier(pWal); - memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); +/* +** Return a pointer to the pPager->pBackup variable. The backup module +** in backup.c maintains the content of this variable. This module +** uses it opaquely as an argument to sqlite3BackupRestart() and +** sqlite3BackupUpdate() only. +*/ +SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ + return &pPager->pBackup; +} - if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ - return 1; /* Dirty read */ - } - if( h1.isInit==0 ){ - return 1; /* Malformed header - probably all zeros */ - } - walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum); - if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){ - return 1; /* Checksum does not match */ +#ifndef SQLITE_OMIT_WAL +/* +** This function is called when the user invokes "PRAGMA wal_checkpoint", +** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint() +** or wal_blocking_checkpoint() API functions. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + rc = sqlite3WalCheckpoint(pPager->pWal, eMode, + pPager->xBusyHandler, pPager->pBusyHandlerArg, + pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pnLog, pnCkpt + ); } + return rc; +} - if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ - *pChanged = 1; - memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); - pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); - testcase( pWal->szPage<=32768 ); - testcase( pWal->szPage>=65536 ); - } +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ + return sqlite3WalCallback(pPager->pWal); +} - /* The header was successfully read. Return zero. */ - return 0; +/* +** Return true if the underlying VFS for the given pager supports the +** primitives necessary for write-ahead logging. +*/ +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ + const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); } /* -** Read the wal-index header from the wal-index and into pWal->hdr. -** If the wal-header appears to be corrupt, try to reconstruct the -** wal-index from the WAL before returning. -** -** Set *pChanged to 1 if the wal-index header value in pWal->hdr is -** changed by this opertion. If pWal->hdr is unchanged, set *pChanged -** to 0. -** -** If the wal-index header is successfully read, return SQLITE_OK. -** Otherwise an SQLite error code. +** Attempt to take an exclusive lock on the database file. If a PENDING lock +** is obtained instead, immediately release it. */ -static int walIndexReadHdr(Wal *pWal, int *pChanged){ +static int pagerExclusiveLock(Pager *pPager){ int rc; /* Return code */ - int badHdr; /* True if a header read failed */ - volatile u32 *page0; /* Chunk of wal-index containing header */ - /* Ensure that page 0 of the wal-index (the page that contains the - ** wal-index header) is mapped. Return early if an error occurs here. - */ - assert( pChanged ); - rc = walIndexPage(pWal, 0, &page0); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ - return rc; - }; - assert( page0 || pWal->writeLock==0 ); + /* If the attempt to grab the exclusive lock failed, release the + ** pending lock that may have been obtained instead. */ + pagerUnlockDb(pPager, SHARED_LOCK); + } - /* If the first page of the wal-index has been mapped, try to read the - ** wal-index header immediately, without holding any lock. This usually - ** works, but may fail if the wal-index header is corrupt or currently - ** being modified by another thread or process. - */ - badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); + return rc; +} - /* If the first attempt failed, it might have been due to a race - ** with a writer. So get a WRITE lock and try again. +/* +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** exclusive-locking mode when this function is called, take an EXCLUSIVE +** lock on the database file and use heap-memory to store the wal-index +** in. Otherwise, use the normal shared-memory. +*/ +static int pagerOpenWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->pWal==0 && pPager->tempFile==0 ); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock); + + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory + ** implementation. Take the exclusive lock now, before opening the WAL + ** file, to make sure this is safe. */ - assert( badHdr==0 || pWal->writeLock==0 ); - if( badHdr && SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; - } - } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); } - /* If the header is read successfully, check the version number to make - ** sure the wal-index was not constructed with some future format that - ** this version of SQLite cannot understand. + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), return an error code. */ - if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ - rc = SQLITE_CANTOPEN_BKPT; + if( rc==SQLITE_OK ){ + rc = sqlite3WalOpen(pPager->pVfs, + pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, &pPager->pWal + ); } return rc; } -/* -** This is the value that walTryBeginRead returns when it needs to -** be retried. -*/ -#define WAL_RETRY (-1) /* -** Attempt to start a read transaction. This might fail due to a race or -** other transient condition. When that happens, it returns WAL_RETRY to -** indicate to the caller that it is safe to retry immediately. -** -** On success return SQLITE_OK. On a permanent failure (such an -** I/O error or an SQLITE_BUSY because another process is running -** recovery) return a positive error code. -** -** The useWal parameter is true to force the use of the WAL and disable -** the case where the WAL is bypassed because it has been completely -** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() -** to make a copy of the wal-index header into pWal->hdr. If the -** wal-index header has changed, *pChanged is set to 1 (as an indication -** to the caller that the local paget cache is obsolete and needs to be -** flushed.) When useWal==1, the wal-index header is assumed to already -** be loaded and the pChanged parameter is unused. -** -** The caller must set the cnt parameter to the number of prior calls to -** this routine during the current read attempt that returned WAL_RETRY. -** This routine will start taking more aggressive measures to clear the -** race conditions after multiple WAL_RETRY returns, and after an excessive -** number of errors will ultimately return SQLITE_PROTOCOL. The -** SQLITE_PROTOCOL return indicates that some other process has gone rogue -** and is not honoring the locking protocol. There is a vanishingly small -** chance that SQLITE_PROTOCOL could be returned because of a run of really -** bad luck when there is lots of contention for the wal-index, but that -** possibility is so small that it can be safely neglected, we believe. +** The caller must be holding a SHARED lock on the database file to call +** this function. ** -** On success, this routine obtains a read lock on -** WAL_READ_LOCK(pWal->readLock). The pWal->readLock integer is -** in the range 0 <= pWal->readLock < WAL_NREADER. If pWal->readLock==(-1) -** that means the Wal does not hold any read lock. The reader must not -** access any database page that is modified by a WAL frame up to and -** including frame number aReadMark[pWal->readLock]. The reader will -** use WAL frames up to and including pWal->hdr.mxFrame if pWal->readLock>0 -** Or if pWal->readLock==0, then the reader will ignore the WAL -** completely and get all content directly from the database file. -** If the useWal parameter is 1 then the WAL will never be ignored and -** this routine will always set pWal->readLock>0 on success. -** When the read transaction is completed, the caller must release the -** lock on WAL_READ_LOCK(pWal->readLock) and set pWal->readLock to -1. +** If the pager passed as the first argument is open on a real database +** file (not a temp file or an in-memory database), and the WAL file +** is not already open, make an attempt to open it now. If successful, +** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** not support the xShmXXX() methods, return an error code. *pbOpen is +** not modified in either case. ** -** This routine uses the nBackfill and aReadMark[] fields of the header -** to select a particular WAL_READ_LOCK() that strives to let the -** checkpoint process do as much work as possible. This routine might -** update values of the aReadMark[] array in the header, but if it does -** so it takes care to hold an exclusive lock on the corresponding -** WAL_READ_LOCK() while changing values. +** If the pager is open on a temp-file (or in-memory database), or if +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK +** without doing anything. */ -static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ - volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ - u32 mxReadMark; /* Largest aReadMark[] value */ - int mxI; /* Index of largest aReadMark[] value */ - int i; /* Loop counter */ - int rc = SQLITE_OK; /* Return code */ +SQLITE_PRIVATE int sqlite3PagerOpenWal( + Pager *pPager, /* Pager object */ + int *pbOpen /* OUT: Set to true if call is a no-op */ +){ + int rc = SQLITE_OK; /* Return code */ - assert( pWal->readLock<0 ); /* Not currently locked */ + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - /* Take steps to avoid spinning forever if there is a protocol error. */ - if( cnt>5 ){ - if( cnt>100 ) return SQLITE_PROTOCOL; - sqlite3OsSleep(pWal->pVfs, 1); - } + if( !pPager->tempFile && !pPager->pWal ){ + if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; - if( !useWal ){ - rc = walIndexReadHdr(pWal, pChanged); - if( rc==SQLITE_BUSY ){ - /* If there is not a recovery running in another thread or process - ** then convert BUSY errors to WAL_RETRY. If recovery is known to - ** be running, convert BUSY to BUSY_RECOVERY. There is a race here - ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY - ** would be technically correct. But the race is benign since with - ** WAL_RETRY this routine will be called again and will probably be - ** right on the second iteration. - */ - if( pWal->apWiData[0]==0 ){ - /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. - ** We assume this is a transient condition, so return WAL_RETRY. The - ** xShmMap() implementation used by the default unix and win32 VFS - ** modules may return SQLITE_BUSY due to a race condition in the - ** code that determines whether or not the shared-memory region - ** must be zeroed before the requested page is returned. - */ - rc = WAL_RETRY; - }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ - walUnlockShared(pWal, WAL_RECOVER_LOCK); - rc = WAL_RETRY; - }else if( rc==SQLITE_BUSY ){ - rc = SQLITE_BUSY_RECOVERY; - } - } - if( rc!=SQLITE_OK ){ - return rc; - } - } + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); - pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ - /* The WAL has been completely backfilled (or it is empty). - ** and can be safely ignored. - */ - rc = walLockShared(pWal, WAL_READ_LOCK(0)); - walShmBarrier(pWal); + rc = pagerOpenWal(pPager); if( rc==SQLITE_OK ){ - if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ - /* It is not safe to allow the reader to continue here if frames - ** may have been appended to the log before READ_LOCK(0) was obtained. - ** When holding READ_LOCK(0), the reader ignores the entire log file, - ** which implies that the database file contains a trustworthy - ** snapshoT. Since holding READ_LOCK(0) prevents a checkpoint from - ** happening, this is usually correct. - ** - ** However, if frames have been appended to the log (or if the log - ** is wrapped and written for that matter) before the READ_LOCK(0) - ** is obtained, that is not necessarily true. A checkpointer may - ** have started to backfill the appended frames but crashed before - ** it finished. Leaving a corrupt image in the database file. - */ - walUnlockShared(pWal, WAL_READ_LOCK(0)); - return WAL_RETRY; - } - pWal->readLock = 0; - return SQLITE_OK; - }else if( rc!=SQLITE_BUSY ){ - return rc; + pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; } + }else{ + *pbOpen = 1; } - /* If we get this far, it means that the reader will want to use - ** the WAL to get at content from recent commits. The job now is - ** to select one of the aReadMark[] entries that is closest to - ** but not exceeding pWal->hdr.mxFrame and lock that entry. + return rc; +} + +/* +** This function is called to close the connection to the log file prior +** to switching from WAL to rollback mode. +** +** Before closing the log file, this function attempts to take an +** EXCLUSIVE lock on the database file. If this cannot be obtained, an +** error (SQLITE_BUSY) is returned and the log connection is not closed. +** If successful, the EXCLUSIVE lock is not released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + + /* If the log file is not already open, but does exist in the file-system, + ** it may need to be checkpointed before the connection can switch to + ** rollback mode. Open it now so this can happen. */ - mxReadMark = 0; - mxI = 0; - for(i=1; iaReadMark[i]; - if( mxReadMark<=thisMark && thisMark<=pWal->hdr.mxFrame ){ - assert( thisMark!=READMARK_NOT_USED ); - mxReadMark = thisMark; - mxI = i; + if( !pPager->pWal ){ + int logexists = 0; + rc = pagerLockDb(pPager, SHARED_LOCK); + if( rc==SQLITE_OK ){ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists + ); + } + if( rc==SQLITE_OK && logexists ){ + rc = pagerOpenWal(pPager); } } - if( mxI==0 ){ - /* If we get here, it means that all of the aReadMark[] entries between - ** 1 and WAL_NREADER-1 are zero. Try to initialize aReadMark[1] to - ** be mxFrame, then retry. - */ - rc = walLockExclusive(pWal, WAL_READ_LOCK(1), 1); + + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on + ** the database file, the log and log-summary files will be deleted. + */ + if( rc==SQLITE_OK && pPager->pWal ){ + rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - pInfo->aReadMark[1] = pWal->hdr.mxFrame; - walUnlockExclusive(pWal, WAL_READ_LOCK(1), 1); - rc = WAL_RETRY; - }else if( rc==SQLITE_BUSY ){ - rc = WAL_RETRY; - } - return rc; - }else{ - if( mxReadMark < pWal->hdr.mxFrame ){ - for(i=1; iaReadMark[i] = pWal->hdr.mxFrame; - mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; - } - } + rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); + pPager->pWal = 0; } + } + return rc; +} - rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); - if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; - } - /* Now that the read-lock has been obtained, check that neither the - ** value in the aReadMark[] array or the contents of the wal-index - ** header have changed. - ** - ** It is necessary to check that the wal-index header did not change - ** between the time it was read and when the shared-lock was obtained - ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility - ** that the log file may have been wrapped by a writer, or that frames - ** that occur later in the log than pWal->hdr.mxFrame may have been - ** copied into the database by a checkpointer. If either of these things - ** happened, then reading the database with the current value of - ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry - ** instead. - ** - ** This does not guarantee that the copy of the wal-index header is up to - ** date before proceeding. That would not be possible without somehow - ** blocking writers. It only guarantees that a dangerous checkpoint or - ** log-wrap (either of which would require an exclusive lock on - ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. - */ - walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark - || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ - walUnlockShared(pWal, WAL_READ_LOCK(mxI)); - return WAL_RETRY; - }else{ - assert( mxReadMark<=pWal->hdr.mxFrame ); - pWal->readLock = (i16)mxI; - } - } - return rc; +#ifdef SQLITE_HAS_CODEC +/* +** This function is called by the wal module when writing page content +** into the log file. +** +** This function returns a pointer to a buffer containing the encrypted +** page content. If a malloc fails, this function may return NULL. +*/ +SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ + void *aData = 0; + CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); + return aData; } +#endif /* SQLITE_HAS_CODEC */ + +#endif /* !SQLITE_OMIT_WAL */ + +#endif /* SQLITE_OMIT_DISKIO */ +/************** End of pager.c ***********************************************/ +/************** Begin file wal.c *********************************************/ /* -** Begin a read transaction on the database. +** 2010 February 1 ** -** This routine used to be called sqlite3OpenSnapshot() and with good reason: -** it takes a snapshot of the state of the WAL and wal-index for the current -** instant in time. The current thread will continue to use this snapshot. -** Other threads might append new content to the WAL and wal-index but -** that extra content is ignored by the current thread. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** If the database contents have changes since the previous read -** transaction, then *pChanged is set to 1 before returning. The -** Pager layer will use this to know that is cache is stale and -** needs to be flushed. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of a write-ahead log (WAL) used in +** "journal_mode=WAL" mode. +** +** WRITE-AHEAD LOG (WAL) FILE FORMAT +** +** A WAL file consists of a header followed by zero or more "frames". +** Each frame records the revised content of a single page from the +** database file. All changes to the database are recorded by writing +** frames into the WAL. Transactions commit when a frame is written that +** contains a commit marker. A single WAL can and usually does record +** multiple transactions. Periodically, the content of the WAL is +** transferred back into the database file in an operation called a +** "checkpoint". +** +** A single WAL file can be used multiple times. In other words, the +** WAL can fill up with frames and then be checkpointed and then new +** frames can overwrite the old ones. A WAL always grows from beginning +** toward the end. Checksums and counters attached to each frame are +** used to determine which frames within the WAL are valid and which +** are leftovers from prior checkpoints. +** +** The WAL header is 32 bytes in size and consists of the following eight +** big-endian 32-bit unsigned integer values: +** +** 0: Magic number. 0x377f0682 or 0x377f0683 +** 4: File format version. Currently 3007000 +** 8: Database page size. Example: 1024 +** 12: Checkpoint sequence number +** 16: Salt-1, random integer incremented with each checkpoint +** 20: Salt-2, a different random integer changing with each ckpt +** 24: Checksum-1 (first part of checksum for first 24 bytes of header). +** 28: Checksum-2 (second part of checksum for first 24 bytes of header). +** +** Immediately following the wal-header are zero or more frames. Each +** frame consists of a 24-byte frame-header followed by a bytes +** of page data. The frame-header is six big-endian 32-bit unsigned +** integer values, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Salt-1 (copied from the header) +** 12: Salt-2 (copied from the header) +** 16: Checksum-1. +** 20: Checksum-2. +** +** A frame is considered valid if and only if the following conditions are +** true: +** +** (1) The salt-1 and salt-2 values in the frame-header match +** salt values in the wal-header +** +** (2) The checksum values in the final 8 bytes of the frame-header +** exactly match the checksum computed consecutively on the +** WAL header and the first 8 bytes and the content of all frames +** up to and including the current frame. +** +** The checksum is computed using 32-bit big-endian integers if the +** magic number in the first 4 bytes of the WAL is 0x377f0683 and it +** is computed using little-endian if the magic number is 0x377f0682. +** The checksum values are always stored in the frame header in a +** big-endian format regardless of which byte order is used to compute +** the checksum. The checksum is computed by interpreting the input as +** an even number of unsigned 32-bit integers: x[0] through x[N]. The +** algorithm used for the checksum is as follows: +** +** for i from 0 to n-1 step 2: +** s0 += x[i] + s1; +** s1 += x[i+1] + s0; +** endfor +** +** Note that s0 and s1 are both weighted checksums using fibonacci weights +** in reverse order (the largest fibonacci weight occurs on the first element +** of the sequence being summed.) The s1 value spans all 32-bit +** terms of the sequence whereas s0 omits the final term. +** +** On a checkpoint, the WAL is first VFS.xSync-ed, then valid content of the +** WAL is transferred into the database, then the database is VFS.xSync-ed. +** The VFS.xSync operations serve as write barriers - all writes launched +** before the xSync must complete before any write that launches after the +** xSync begins. +** +** After each checkpoint, the salt-1 value is incremented and the salt-2 +** value is randomized. This prevents old and new frames in the WAL from +** being considered valid at the same time and being checkpointing together +** following a crash. +** +** READER ALGORITHM +** +** To read a page from the database (call it page number P), a reader +** first checks the WAL to see if it contains page P. If so, then the +** last valid instance of page P that is a followed by a commit frame +** or is a commit frame itself becomes the value read. If the WAL +** contains no copies of page P that are valid and which are a commit +** frame or are followed by a commit frame, then page P is read from +** the database file. +** +** To start a read transaction, the reader records the index of the last +** valid frame in the WAL. The reader uses this recorded "mxFrame" value +** for all subsequent read operations. New transactions can be appended +** to the WAL, but as long as the reader uses its original mxFrame value +** and ignores the newly appended content, it will see a consistent snapshot +** of the database from a single point in time. This technique allows +** multiple concurrent readers to view different versions of the database +** content simultaneously. +** +** The reader algorithm in the previous paragraphs works correctly, but +** because frames for page P can appear anywhere within the WAL, the +** reader has to scan the entire WAL looking for page P frames. If the +** WAL is large (multiple megabytes is typical) that scan can be slow, +** and read performance suffers. To overcome this problem, a separate +** data structure called the wal-index is maintained to expedite the +** search for frames of a particular page. +** +** WAL-INDEX FORMAT +** +** Conceptually, the wal-index is shared memory, though VFS implementations +** might choose to implement the wal-index using a mmapped file. Because +** the wal-index is shared memory, SQLite does not support journal_mode=WAL +** on a network filesystem. All users of the database must be able to +** share memory. +** +** The wal-index is transient. After a crash, the wal-index can (and should +** be) reconstructed from the original WAL file. In fact, the VFS is required +** to either truncate or zero the header of the wal-index when the last +** connection to it closes. Because the wal-index is transient, it can +** use an architecture-specific format; it does not have to be cross-platform. +** Hence, unlike the database and WAL file formats which store all values +** as big endian, the wal-index can store multi-byte values in the native +** byte order of the host computer. +** +** The purpose of the wal-index is to answer this question quickly: Given +** a page number P, return the index of the last frame for page P in the WAL, +** or return NULL if there are no frames for page P in the WAL. +** +** The wal-index consists of a header region, followed by an one or +** more index blocks. +** +** The wal-index header contains the total number of frames within the WAL +** in the the mxFrame field. +** +** Each index block except for the first contains information on +** HASHTABLE_NPAGE frames. The first index block contains information on +** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and +** HASHTABLE_NPAGE are selected so that together the wal-index header and +** first index block are the same size as all other index blocks in the +** wal-index. +** +** Each index block contains two sections, a page-mapping that contains the +** database page number associated with each wal frame, and a hash-table +** that allows readers to query an index block for a specific page number. +** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE +** for the first index block) 32-bit page numbers. The first entry in the +** first index-block contains the database page number corresponding to the +** first frame in the WAL file. The first entry in the second index block +** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in +** the log, and so on. +** +** The last index block in a wal-index usually contains less than the full +** complement of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE) page-numbers, +** depending on the contents of the WAL file. This does not change the +** allocated size of the page-mapping array - the page-mapping array merely +** contains unused entries. +** +** Even without using the hash table, the last frame for page P +** can be found by scanning the page-mapping sections of each index block +** starting with the last index block and moving toward the first, and +** within each index block, starting at the end and moving toward the +** beginning. The first entry that equals P corresponds to the frame +** holding the content for that page. +** +** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers. +** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the +** hash table for each page number in the mapping section, so the hash +** table is never more than half full. The expected number of collisions +** prior to finding a match is 1. Each entry of the hash table is an +** 1-based index of an entry in the mapping section of the same +** index block. Let K be the 1-based index of the largest entry in +** the mapping section. (For index blocks other than the last, K will +** always be exactly HASHTABLE_NPAGE (4096) and for the last index block +** K will be (mxFrame%HASHTABLE_NPAGE).) Unused slots of the hash table +** contain a value of 0. +** +** To look for page P in the hash table, first compute a hash iKey on +** P as follows: +** +** iKey = (P * 383) % HASHTABLE_NSLOT +** +** Then start scanning entries of the hash table, starting with iKey +** (wrapping around to the beginning when the end of the hash table is +** reached) until an unused hash slot is found. Let the first unused slot +** be at index iUnused. (iUnused might be less than iKey if there was +** wrap-around.) Because the hash table is never more than half full, +** the search is guaranteed to eventually hit an unused entry. Let +** iMax be the value between iKey and iUnused, closest to iUnused, +** where aHash[iMax]==P. If there is no iMax entry (if there exists +** no hash slot such that aHash[i]==p) then page P is not in the +** current index block. Otherwise the iMax-th mapping entry of the +** current index block corresponds to the last entry that references +** page P. +** +** A hash search begins with the last index block and moves toward the +** first index block, looking for entries corresponding to page P. On +** average, only two or three slots in each index block need to be +** examined in order to either find the last entry for page P, or to +** establish that no such entry exists in the block. Each index block +** holds over 4000 entries. So two or three index blocks are sufficient +** to cover a typical 10 megabyte WAL file, assuming 1K pages. 8 or 10 +** comparisons (on average) suffice to either locate a frame in the +** WAL or to establish that the frame does not exist in the WAL. This +** is much faster than scanning the entire 10MB WAL. +** +** Note that entries are added in order of increasing K. Hence, one +** reader might be using some value K0 and a second reader that started +** at a later time (after additional transactions were added to the WAL +** and to the wal-index) might be using a different value K1, where K1>K0. +** Both readers can use the same hash table and mapping section to get +** the correct result. There may be entries in the hash table with +** K>K0 but to the first reader, those entries will appear to be unused +** slots in the hash table and so the first reader will get an answer as +** if no values greater than K0 had ever been inserted into the hash table +** in the first place - which is what reader one wants. Meanwhile, the +** second reader using K1 will see additional values that were inserted +** later, which is exactly what reader two wants. +** +** When a rollback occurs, the value of K is decreased. Hash table entries +** that correspond to frames greater than the new K value are removed +** from the hash table at this point. */ -SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ - int rc; /* Return code */ - int cnt = 0; /* Number of TryBeginRead attempts */ +#ifndef SQLITE_OMIT_WAL - do{ - rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); - }while( rc==WAL_RETRY ); - return rc; -} /* -** Finish with a read transaction. All this does is release the -** read-lock. +** Trace output macros */ -SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ - sqlite3WalEndWriteTransaction(pWal); - if( pWal->readLock>=0 ){ - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->readLock = -1; - } -} +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3WalTrace = 0; +# define WALTRACE(X) if(sqlite3WalTrace) sqlite3DebugPrintf X +#else +# define WALTRACE(X) +#endif /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** The maximum (and only) versions of the wal and wal-index formats +** that may be interpreted by this version of SQLite. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** If a client begins recovering a WAL file and finds that (a) the checksum +** values in the wal-header are correct and (b) the version field is not +** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN. +** +** Similarly, if a client successfully reads a wal-index header (i.e. the +** checksum test is successful) and finds that the version field is not +** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite +** returns SQLITE_CANTOPEN. */ -SQLITE_PRIVATE int sqlite3WalRead( - Wal *pWal, /* WAL handle */ - Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ -){ - u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ - int iHash; /* Used to loop through N hash tables */ - - /* This routine is only be called from within a read transaction. */ - assert( pWal->readLock>=0 || pWal->lockError ); - - /* If the "last page" field of the wal-index header snapshot is 0, then - ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the - ** WAL were empty. - */ - if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; - return SQLITE_OK; - } - - /* Search the hash table or tables for an entry matching page number - ** pgno. Each iteration of the following for() loop searches one - ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). - ** - ** This code might run concurrently to the code in walIndexAppend() - ** that adds entries to the wal-index (and possibly to this hash - ** table). This means the value just read from the hash - ** slot (aHash[iKey]) may have been added before or after the - ** current read transaction was opened. Values added after the - ** read transaction was opened may have been written incorrectly - - ** i.e. these slots may contain garbage data. However, we assume - ** that any slots written before the current read transaction was - ** opened remain unmodified. - ** - ** For the reasons above, the if(...) condition featured in the inner - ** loop of the following block is more stringent that would be required - ** if we had exclusive access to the hash-table: - ** - ** (aPgno[iFrame]==pgno): - ** This condition filters out normal hash-table collisions. - ** - ** (iFrame<=iLast): - ** This condition filters out entries that were added to the hash - ** table after the current read-transaction had started. - */ - for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ - volatile ht_slot *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ - u32 iZero; /* Frame number corresponding to aPgno[0] */ - int iKey; /* Hash slot index */ - int nCollide; /* Number of hash collisions remaining */ - int rc; /* Error code */ - - rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); - if( rc!=SQLITE_OK ){ - return rc; - } - nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ - assert( iFrame>iRead ); - iRead = iFrame; - } - if( (nCollide--)==0 ){ - return SQLITE_CORRUPT_BKPT; - } - } - } - -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT - /* If expensive assert() statements are available, do a linear search - ** of the wal-index file content. Make sure the results agree with the - ** result obtained using the hash indexes above. */ - { - u32 iRead2 = 0; - u32 iTest; - for(iTest=iLast; iTest>0; iTest--){ - if( walFramePgno(pWal, iTest)==pgno ){ - iRead2 = iTest; - break; - } - } - assert( iRead==iRead2 ); - } -#endif - - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); - } +#define WAL_MAX_VERSION 3007000 +#define WALINDEX_MAX_VERSION 3007000 - *pInWal = 0; - return SQLITE_OK; -} +/* +** Indices of various locking bytes. WAL_NREADER is the number +** of available reader locks and should be at least 3. +*/ +#define WAL_WRITE_LOCK 0 +#define WAL_ALL_BUT_WRITE 1 +#define WAL_CKPT_LOCK 1 +#define WAL_RECOVER_LOCK 2 +#define WAL_READ_LOCK(I) (3+(I)) +#define WAL_NREADER (SQLITE_SHM_NLOCK-3) -/* -** Return the size of the database in pages (or zero, if unknown). -*/ -SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ - if( pWal && ALWAYS(pWal->readLock>=0) ){ - return pWal->hdr.nPage; - } - return 0; -} +/* Object declarations */ +typedef struct WalIndexHdr WalIndexHdr; +typedef struct WalIterator WalIterator; +typedef struct WalCkptInfo WalCkptInfo; -/* -** This function starts a write transaction on the WAL. -** -** A read transaction must have already been started by a prior call -** to sqlite3WalBeginReadTransaction(). +/* +** The following object holds a copy of the wal-index header content. ** -** If another thread or process has written into the database since -** the read transaction was started, then it is not possible for this -** thread to write as doing so would cause a fork. So this routine -** returns SQLITE_BUSY in that case and no write transaction is started. +** The actual header in the wal-index consists of two copies of this +** object. ** -** There can only be a single writer active at a time. +** The szPage value can be any power of 2 between 512 and 32768, inclusive. +** Or it can be 1 to represent a 65536-byte page. The latter case was +** added in 3.7.1 when support for 64K pages was added. */ -SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ - int rc; +struct WalIndexHdr { + u32 iVersion; /* Wal-index version */ + u32 unused; /* Unused (padding) field */ + u32 iChange; /* Counter incremented each transaction */ + u8 isInit; /* 1 when initialized */ + u8 bigEndCksum; /* True if checksums in WAL are big-endian */ + u16 szPage; /* Database page size in bytes. 1==64K */ + u32 mxFrame; /* Index of last valid frame in the WAL */ + u32 nPage; /* Size of database in pages */ + u32 aFrameCksum[2]; /* Checksum of last frame in log */ + u32 aSalt[2]; /* Two salt values copied from WAL header */ + u32 aCksum[2]; /* Checksum over all prior fields */ +}; - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock>=0 ); +/* +** A copy of the following object occurs in the wal-index immediately +** following the second copy of the WalIndexHdr. This object stores +** information used by checkpoint. +** +** nBackfill is the number of frames in the WAL that have been written +** back into the database. (We call the act of moving content from WAL to +** database "backfilling".) The nBackfill number is never greater than +** WalIndexHdr.mxFrame. nBackfill can only be increased by threads +** holding the WAL_CKPT_LOCK lock (which includes a recovery thread). +** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from +** mxFrame back to zero when the WAL is reset. +** +** There is one entry in aReadMark[] for each reader lock. If a reader +** holds read-lock K, then the value in aReadMark[K] is no greater than +** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) +** for any aReadMark[] means that entry is unused. aReadMark[0] is +** a special case; its value is never used and it exists as a place-holder +** to avoid having to offset aReadMark[] indexs by one. Readers holding +** WAL_READ_LOCK(0) always ignore the entire WAL and read all content +** directly from the database. +** +** The value of aReadMark[K] may only be changed by a thread that +** is holding an exclusive lock on WAL_READ_LOCK(K). Thus, the value of +** aReadMark[K] cannot changed while there is a reader is using that mark +** since the reader will be holding a shared lock on WAL_READ_LOCK(K). +** +** The checkpointer may only transfer frames from WAL to database where +** the frame numbers are less than or equal to every aReadMark[] that is +** in use (that is, every aReadMark[j] for which there is a corresponding +** WAL_READ_LOCK(j)). New readers (usually) pick the aReadMark[] with the +** largest value and will increase an unused aReadMark[] to mxFrame if there +** is not already an aReadMark[] equal to mxFrame. The exception to the +** previous sentence is when nBackfill equals mxFrame (meaning that everything +** in the WAL has been backfilled into the database) then new readers +** will choose aReadMark[0] which has value 0 and hence such reader will +** get all their all content directly from the database file and ignore +** the WAL. +** +** Writers normally append new frames to the end of the WAL. However, +** if nBackfill equals mxFrame (meaning that all WAL content has been +** written back into the database) and if no readers are using the WAL +** (in other words, if there are no WAL_READ_LOCK(i) where i>0) then +** the writer will first "reset" the WAL back to the beginning and start +** writing new content beginning at frame 1. +** +** We assume that 32-bit loads are atomic and so no locks are needed in +** order to read from any aReadMark[] entries. +*/ +struct WalCkptInfo { + u32 nBackfill; /* Number of WAL frames backfilled into DB */ + u32 aReadMark[WAL_NREADER]; /* Reader marks */ +}; +#define READMARK_NOT_USED 0xffffffff - if( pWal->readOnly ){ - return SQLITE_READONLY; - } - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. - */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - if( rc ){ - return rc; - } - pWal->writeLock = 1; +/* A block of WALINDEX_LOCK_RESERVED bytes beginning at +** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems +** only support mandatory file-locks, we do not read or write data +** from the region of the file on which locks are applied. +*/ +#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2 + sizeof(WalCkptInfo)) +#define WALINDEX_LOCK_RESERVED 16 +#define WALINDEX_HDR_SIZE (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED) - /* If another connection has written to the database file since the - ** time the read transaction on this connection was started, then - ** the write is disallowed. - */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - rc = SQLITE_BUSY; - } +/* Size of header before each frame in wal */ +#define WAL_FRAME_HDRSIZE 24 - return rc; -} +/* Size of write ahead log header, including checksum. */ +/* #define WAL_HDRSIZE 24 */ +#define WAL_HDRSIZE 32 + +/* WAL magic value. Either this value, or the same value with the least +** significant bit also set (WAL_MAGIC | 0x00000001) is stored in 32-bit +** big-endian format in the first 4 bytes of a WAL file. +** +** If the LSB is set, then the checksums for each frame within the WAL +** file are calculated by treating all data as an array of 32-bit +** big-endian words. Otherwise, they are calculated by interpreting +** all data as 32-bit little-endian words. +*/ +#define WAL_MAGIC 0x377f0682 /* -** End a write transaction. The commit has already been done. This -** routine merely releases the lock. +** Return the offset of frame iFrame in the write-ahead log file, +** assuming a database page size of szPage bytes. The offset returned +** is to the start of the write-ahead log frame-header. */ -SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ - if( pWal->writeLock ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - } - return SQLITE_OK; -} +#define walFrameOffset(iFrame, szPage) ( \ + WAL_HDRSIZE + ((iFrame)-1)*(i64)((szPage)+WAL_FRAME_HDRSIZE) \ +) /* -** If any data has been written (but not committed) to the log file, this -** function moves the write-pointer back to the start of the transaction. -** -** Additionally, the callback function is invoked for each frame written -** to the WAL since the start of the transaction. If the callback returns -** other than SQLITE_OK, it is not invoked again and the error code is -** returned to the caller. -** -** Otherwise, if the callback function does not return an error, this -** function returns SQLITE_OK. +** An open write-ahead log file is represented by an instance of the +** following object. */ -SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ - int rc = SQLITE_OK; - if( ALWAYS(pWal->writeLock) ){ - Pgno iMax = pWal->hdr.mxFrame; - Pgno iFrame; - - /* Restore the clients cache of the wal-index header to the state it - ** was in before the client began writing to the database. - */ - memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); +struct Wal { + sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ + sqlite3_file *pDbFd; /* File handle for the database file */ + sqlite3_file *pWalFd; /* File handle for WAL file */ + u32 iCallback; /* Value to pass to log callback (or 0) */ + i64 mxWalSize; /* Truncate WAL to this size upon reset */ + int nWiData; /* Size of array apWiData */ + volatile u32 **apWiData; /* Pointer to wal-index content in memory */ + u32 szPage; /* Database page size */ + i16 readLock; /* Which read lock is being held. -1 for none */ + u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ + u8 writeLock; /* True if in a write transaction */ + u8 ckptLock; /* True if holding a checkpoint lock */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ + WalIndexHdr hdr; /* Wal-index header for current transaction */ + const char *zWalName; /* Name of WAL file */ + u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ +#ifdef SQLITE_DEBUG + u8 lockError; /* True if a locking error has occurred */ +#endif +}; - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; - iFrame++ - ){ - /* This call cannot fail. Unless the page for which the page number - ** is passed as the second argument is (a) in the cache and - ** (b) has an outstanding reference, then xUndo is either a no-op - ** (if (a) is false) or simply expels the page from the cache (if (b) - ** is false). - ** - ** If the upper layer is doing a rollback, it is guaranteed that there - ** are no outstanding references to any page other than page 1. And - ** page 1 is never written to the log until the transaction is - ** committed. As a result, the call to xUndo may not fail. - */ - assert( walFramePgno(pWal, iFrame)!=1 ); - rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); - } - walCleanupHash(pWal); - } - assert( rc==SQLITE_OK ); - return rc; -} +/* +** Candidate values for Wal.exclusiveMode. +*/ +#define WAL_NORMAL_MODE 0 +#define WAL_EXCLUSIVE_MODE 1 +#define WAL_HEAPMEMORY_MODE 2 -/* -** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 -** values. This function populates the array with values required to -** "rollback" the write position of the WAL handle back to the current -** point in the event of a savepoint rollback (via WalSavepointUndo()). +/* +** Possible values for WAL.readOnly */ -SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ - assert( pWal->writeLock ); - aWalData[0] = pWal->hdr.mxFrame; - aWalData[1] = pWal->hdr.aFrameCksum[0]; - aWalData[2] = pWal->hdr.aFrameCksum[1]; - aWalData[3] = pWal->nCkpt; -} +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ -/* -** Move the write position of the WAL back to the point identified by -** the values in the aWalData[] array. aWalData must point to an array -** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated -** by a call to WalSavepoint(). +/* +** Each page of the wal-index mapping contains a hash-table made up of +** an array of HASHTABLE_NSLOT elements of the following type. */ -SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ - int rc = SQLITE_OK; +typedef u16 ht_slot; - assert( pWal->writeLock ); - assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); +/* +** This structure is used to implement an iterator that loops through +** all frames in the WAL in database page order. Where two or more frames +** correspond to the same database page, the iterator visits only the +** frame most recently written to the WAL (in other words, the frame with +** the largest index). +** +** The internals of this structure are only accessed by: +** +** walIteratorInit() - Create a new iterator, +** walIteratorNext() - Step an iterator, +** walIteratorFree() - Free an iterator. +** +** This functionality is used by the checkpoint code (see walCheckpoint()). +*/ +struct WalIterator { + int iPrior; /* Last result returned from the iterator */ + int nSegment; /* Number of entries in aSegment[] */ + struct WalSegment { + int iNext; /* Next slot in aIndex[] not yet returned */ + ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ + u32 *aPgno; /* Array of page numbers. */ + int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ + int iZero; /* Frame number associated with aPgno[0] */ + } aSegment[1]; /* One for every 32KB page in the wal-index */ +}; - if( aWalData[3]!=pWal->nCkpt ){ - /* This savepoint was opened immediately after the write-transaction - ** was started. Right after that, the writer decided to wrap around - ** to the start of the log. Update the savepoint values to match. - */ - aWalData[0] = 0; - aWalData[3] = pWal->nCkpt; - } +/* +** Define the parameters of the hash tables in the wal-index file. There +** is a hash-table following every HASHTABLE_NPAGE page numbers in the +** wal-index. +** +** Changing any of these constants will alter the wal-index format and +** create incompatibilities. +*/ +#define HASHTABLE_NPAGE 4096 /* Must be power of 2 */ +#define HASHTABLE_HASH_1 383 /* Should be prime */ +#define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ - if( aWalData[0]hdr.mxFrame ){ - pWal->hdr.mxFrame = aWalData[0]; - pWal->hdr.aFrameCksum[0] = aWalData[1]; - pWal->hdr.aFrameCksum[1] = aWalData[2]; - walCleanupHash(pWal); - } +/* +** The block of page numbers associated with the first hash-table in a +** wal-index is smaller than usual. This is so that there is a complete +** hash-table on each aligned 32KB page of the wal-index. +*/ +#define HASHTABLE_NPAGE_ONE (HASHTABLE_NPAGE - (WALINDEX_HDR_SIZE/sizeof(u32))) - return rc; -} +/* The wal-index is divided into pages of WALINDEX_PGSZ bytes each. */ +#define WALINDEX_PGSZ ( \ + sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ +) /* -** This function is called just before writing a set of frames to the log -** file (see sqlite3WalFrames()). It checks to see if, instead of appending -** to the current log file, it is possible to overwrite the start of the -** existing log file with the new frames (i.e. "reset" the log). If so, -** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left -** unchanged. +** Obtain a pointer to the iPage'th page of the wal-index. The wal-index +** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are +** numbered from zero. ** -** SQLITE_OK is returned if no error is encountered (regardless of whether -** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned -** if an error occurs. +** If this call is successful, *ppPage is set to point to the wal-index +** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, +** then an SQLite error code is returned and *ppPage is set to 0. */ -static int walRestartLog(Wal *pWal){ +static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ int rc = SQLITE_OK; - int cnt; - if( pWal->readLock==0 ){ - volatile WalCkptInfo *pInfo = walCkptInfo(pWal); - assert( pInfo->nBackfill==pWal->hdr.mxFrame ); - if( pInfo->nBackfill>0 ){ - rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); - if( rc==SQLITE_OK ){ - /* If all readers are using WAL_READ_LOCK(0) (in other words if no - ** readers are currently using the WAL), then the transactions - ** frames will overwrite the start of the existing log. Update the - ** wal-index header to reflect this. - ** - ** In theory it would be Ok to update the cache of the header only - ** at this point. But updating the actual wal-index header is also - ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - sqlite3_randomness(4, &aSalt[1]); - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - for(i=1; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); - walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); - }else if( rc!=SQLITE_BUSY ){ - return rc; + /* Enlarge the pWal->apWiData[] array if required */ + if( pWal->nWiData<=iPage ){ + int nByte = sizeof(u32*)*(iPage+1); + volatile u32 **apNew; + apNew = (volatile u32 **)sqlite3_realloc((void *)pWal->apWiData, nByte); + if( !apNew ){ + *ppPage = 0; + return SQLITE_NOMEM; + } + memset((void*)&apNew[pWal->nWiData], 0, + sizeof(u32*)*(iPage+1-pWal->nWiData)); + pWal->apWiData = apNew; + pWal->nWiData = iPage+1; + } + + /* Request a pointer to the required page from the VFS */ + if( pWal->apWiData[iPage]==0 ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + if( rc==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + rc = SQLITE_OK; } } - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = -1; - cnt = 0; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); - }while( rc==WAL_RETRY ); } + + *ppPage = pWal->apWiData[iPage]; + assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); return rc; } -/* -** Write a set of frames to the log. The caller must hold the write-lock -** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +/* +** Return a pointer to the WalCkptInfo structure in the wal-index. */ -SQLITE_PRIVATE int sqlite3WalFrames( - Wal *pWal, /* Wal handle to write to */ - int szPage, /* Database page-size in bytes */ - PgHdr *pList, /* List of dirty pages to write */ - Pgno nTruncate, /* Database size after this commit */ - int isCommit, /* True if this is a commit */ - int sync_flags /* Flags to pass to OsSync() (or 0) */ -){ - int rc; /* Used to catch return codes */ - u32 iFrame; /* Next frame address */ - u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ - PgHdr *p; /* Iterator to run through pList with. */ - PgHdr *pLast = 0; /* Last frame in list */ - int nLast = 0; /* Number of extra copies of last page */ +static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); +} - assert( pList ); - assert( pWal->writeLock ); +/* +** Return a pointer to the WalIndexHdr structure in the wal-index. +*/ +static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalIndexHdr*)pWal->apWiData[0]; +} -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) - { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} - WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", - pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill")); - } -#endif +/* +** The argument to this macro must be of type u32. On a little-endian +** architecture, it returns the u32 value that results from interpreting +** the 4 bytes as a big-endian value. On a big-endian architecture, it +** returns the value that would be produced by intepreting the 4 bytes +** of the input value as a little-endian integer. +*/ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) - /* See if it is possible to write these frames into the start of the - ** log file, instead of appending to it at pWal->hdr.mxFrame. - */ - if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ - return rc; - } +/* +** Generate or extend an 8 byte checksum based on the data in +** array aByte[] and the initial values of aIn[0] and aIn[1] (or +** initial values of 0 and 0 if aIn==NULL). +** +** The checksum is written back into aOut[] before returning. +** +** nByte must be a positive multiple of 8. +*/ +static void walChecksumBytes( + int nativeCksum, /* True for native byte-order, false for non-native */ + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + const u32 *aIn, /* Initial checksum value input */ + u32 *aOut /* OUT: Final checksum value output */ +){ + u32 s1, s2; + u32 *aData = (u32 *)a; + u32 *aEnd = (u32 *)&a[nByte]; - /* If this is the first frame written into the log, write the WAL - ** header to the start of the WAL file. See comments at the top of - ** this source file for a description of the WAL header format. - */ - iFrame = pWal->hdr.mxFrame; - if( iFrame==0 ){ - u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ - u32 aCksum[2]; /* Checksum for wal-header */ + if( aIn ){ + s1 = aIn[0]; + s2 = aIn[1]; + }else{ + s1 = s2 = 0; + } - sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); - sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); - sqlite3Put4byte(&aWalHdr[8], szPage); - sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - sqlite3_randomness(8, pWal->hdr.aSalt); - memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); - walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); - sqlite3Put4byte(&aWalHdr[24], aCksum[0]); - sqlite3Put4byte(&aWalHdr[28], aCksum[1]); - - pWal->szPage = szPage; - pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; - pWal->hdr.aFrameCksum[0] = aCksum[0]; - pWal->hdr.aFrameCksum[1] = aCksum[1]; + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); - rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); - WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); - if( rc!=SQLITE_OK ){ - return rc; - } + if( nativeCksum ){ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aDataszPage==szPage ); - /* Write the log file. */ - for(p=pList; p; p=p->pDirty){ - u32 nDbsize; /* Db-size field for frame header */ - i64 iOffset; /* Write offset in log file */ - void *pData; - - iOffset = walFrameOffset(++iFrame, szPage); - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - - /* Populate and write the frame header */ - nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0; -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; -#else - pData = p->pData; -#endif - walEncodeFrame(pWal, p->pgno, nDbsize, pData, aFrame); - rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset); - if( rc!=SQLITE_OK ){ - return rc; - } + aOut[0] = s1; + aOut[1] = s2; +} - /* Write the page data */ - rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset+sizeof(aFrame)); - if( rc!=SQLITE_OK ){ - return rc; - } - pLast = p; +static void walShmBarrier(Wal *pWal){ + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmBarrier(pWal->pDbFd); } +} - /* Sync the log file if the 'isSync' flag was specified. */ - if( sync_flags ){ - i64 iSegment = sqlite3OsSectorSize(pWal->pWalFd); - i64 iOffset = walFrameOffset(iFrame+1, szPage); - - assert( isCommit ); - assert( iSegment>0 ); +/* +** Write the header information in pWal->hdr into the wal-index. +** +** The checksum on pWal->hdr is updated before it is written. +*/ +static void walIndexWriteHdr(Wal *pWal){ + volatile WalIndexHdr *aHdr = walIndexHdr(pWal); + const int nCksum = offsetof(WalIndexHdr, aCksum); - iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment); - while( iOffsetpData; -#endif - walEncodeFrame(pWal, pLast->pgno, nTruncate, pData, aFrame); - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset); - if( rc!=SQLITE_OK ){ - return rc; - } - iOffset += WAL_FRAME_HDRSIZE; - rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset); - if( rc!=SQLITE_OK ){ - return rc; - } - nLast++; - iOffset += szPage; - } + assert( pWal->writeLock ); + pWal->hdr.isInit = 1; + pWal->hdr.iVersion = WALINDEX_MAX_VERSION; + walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + walShmBarrier(pWal); + memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); +} - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); +/* +** This function encodes a single frame header and writes it to a buffer +** supplied by the caller. A frame-header is made up of a series of +** 4-byte big-endian integers, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Salt-1 (copied from the wal-header) +** 12: Salt-2 (copied from the wal-header) +** 16: Checksum-1. +** 20: Checksum-2. +*/ +static void walEncodeFrame( + Wal *pWal, /* The write-ahead log */ + u32 iPage, /* Database page number for frame */ + u32 nTruncate, /* New db size (or 0 for non-commit frames) */ + u8 *aData, /* Pointer to page data */ + u8 *aFrame /* OUT: Write encoded frame here */ +){ + int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; + assert( WAL_FRAME_HDRSIZE==24 ); + sqlite3Put4byte(&aFrame[0], iPage); + sqlite3Put4byte(&aFrame[4], nTruncate); + memcpy(&aFrame[8], pWal->hdr.aSalt, 8); + + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + + sqlite3Put4byte(&aFrame[16], aCksum[0]); + sqlite3Put4byte(&aFrame[20], aCksum[1]); +} + +/* +** Check to see if the frame with header in aFrame[] and content +** in aData[] is valid. If it is a valid frame, fill *piPage and +** *pnTruncate and return true. Return if the frame is not valid. +*/ +static int walDecodeFrame( + Wal *pWal, /* The write-ahead log */ + u32 *piPage, /* OUT: Database page number for frame */ + u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ + u8 *aData, /* Pointer to page data (for checksum) */ + u8 *aFrame /* Frame data */ +){ + int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; + u32 pgno; /* Page number of the frame */ + assert( WAL_FRAME_HDRSIZE==24 ); + + /* A frame is only valid if the salt values in the frame-header + ** match the salt values in the wal-header. + */ + if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){ + return 0; } - /* Append data to the wal-index. It is not necessary to lock the - ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index - ** guarantees that there are no other writers, and no data that may - ** be in use by existing readers is being overwritten. + /* A frame is only valid if the page number is creater than zero. */ - iFrame = pWal->hdr.mxFrame; - for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ - iFrame++; - rc = walIndexAppend(pWal, iFrame, p->pgno); + pgno = sqlite3Get4byte(&aFrame[0]); + if( pgno==0 ){ + return 0; } - while( nLast>0 && rc==SQLITE_OK ){ - iFrame++; - nLast--; - rc = walIndexAppend(pWal, iFrame, pLast->pgno); + + /* A frame is only valid if a checksum of the WAL header, + ** all prior frams, the first 16 bytes of this frame-header, + ** and the frame-data matches the checksum in the last 8 + ** bytes of this frame-header. + */ + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) + || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) + ){ + /* Checksum failed. */ + return 0; } - if( rc==SQLITE_OK ){ - /* Update the private copy of the header. */ - pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - pWal->hdr.mxFrame = iFrame; - if( isCommit ){ - pWal->hdr.iChange++; - pWal->hdr.nPage = nTruncate; - } - /* If this is a commit, update the wal-index header too. */ - if( isCommit ){ - walIndexWriteHdr(pWal); - pWal->iCallback = iFrame; - } + /* If we reach this point, the frame is valid. Return the page number + ** and the new database size. + */ + *piPage = pgno; + *pnTruncate = sqlite3Get4byte(&aFrame[4]); + return 1; +} + + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +/* +** Names of locks. This routine is used to provide debugging output and is not +** a part of an ordinary build. +*/ +static const char *walLockName(int lockIdx){ + if( lockIdx==WAL_WRITE_LOCK ){ + return "WRITE-LOCK"; + }else if( lockIdx==WAL_CKPT_LOCK ){ + return "CKPT-LOCK"; + }else if( lockIdx==WAL_RECOVER_LOCK ){ + return "RECOVER-LOCK"; + }else{ + static char zName[15]; + sqlite3_snprintf(sizeof(zName), zName, "READ-LOCK[%d]", + lockIdx-WAL_READ_LOCK(0)); + return zName; } +} +#endif /*defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ + - WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); +/* +** Set or release locks on the WAL. Locks are either shared or exclusive. +** A lock cannot be moved directly between shared and exclusive - it must go +** through the unlocked state first. +** +** In locking_mode=EXCLUSIVE, all of these routines become no-ops. +*/ +static int walLockShared(Wal *pWal, int lockIdx){ + int rc; + if( pWal->exclusiveMode ) return SQLITE_OK; + rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, + walLockName(lockIdx), rc ? "failed" : "ok")); + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + return rc; +} +static void walUnlockShared(Wal *pWal, int lockIdx){ + if( pWal->exclusiveMode ) return; + (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); +} +static int walLockExclusive(Wal *pWal, int lockIdx, int n){ + int rc; + if( pWal->exclusiveMode ) return SQLITE_OK; + rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, + walLockName(lockIdx), n, rc ? "failed" : "ok")); + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) return rc; } +static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ + if( pWal->exclusiveMode ) return; + (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal, + walLockName(lockIdx), n)); +} + +/* +** Compute a hash on a page number. The resulting hash value must land +** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances +** the hash to the next value in the event of a collision. +*/ +static int walHash(u32 iPage){ + assert( iPage>0 ); + assert( (HASHTABLE_NSLOT & (HASHTABLE_NSLOT-1))==0 ); + return (iPage*HASHTABLE_HASH_1) & (HASHTABLE_NSLOT-1); +} +static int walNextHash(int iPriorHash){ + return (iPriorHash+1)&(HASHTABLE_NSLOT-1); +} /* -** This routine is called to implement sqlite3_wal_checkpoint() and -** related interfaces. +** Return pointers to the hash table and page number array stored on +** page iHash of the wal-index. The wal-index is broken into 32KB pages +** numbered starting from 0. ** -** Obtain a CHECKPOINT lock and then backfill as much information as -** we can from WAL into the database. +** Set output variable *paHash to point to the start of the hash table +** in the wal-index file. Set *piZero to one less than the frame +** number of the first frame indexed by this hash table. If a +** slot in the hash table is set to N, it refers to frame number +** (*piZero+N) in the log. +** +** Finally, set *paPgno so that *paPgno[1] is the page number of the +** first frame indexed by the hash table, frame (*piZero+1). */ -SQLITE_PRIVATE int sqlite3WalCheckpoint( - Wal *pWal, /* Wal connection */ - int sync_flags, /* Flags to sync db file with (or 0) */ - int nBuf, /* Size of temporary buffer */ - u8 *zBuf /* Temporary buffer to use */ +static int walHashGet( + Wal *pWal, /* WAL handle */ + int iHash, /* Find the iHash'th table */ + volatile ht_slot **paHash, /* OUT: Pointer to hash index */ + volatile u32 **paPgno, /* OUT: Pointer to page number array */ + u32 *piZero /* OUT: Frame associated with *paPgno[0] */ ){ int rc; /* Return code */ - int isChanged = 0; /* True if a new wal-index header is loaded */ - - assert( pWal->ckptLock==0 ); + volatile u32 *aPgno; - WALTRACE(("WAL%p: checkpoint begins\n", pWal)); - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); - if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ - return rc; - } - pWal->ckptLock = 1; + rc = walIndexPage(pWal, iHash, &aPgno); + assert( rc==SQLITE_OK || iHash>0 ); - /* Copy data from the log to the database file. */ - rc = walIndexReadHdr(pWal, &isChanged); if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, sync_flags, nBuf, zBuf); - } - if( isChanged ){ - /* If a new wal-index header was loaded before the checkpoint was - ** performed, then the pager-cache associated with pWal is now - ** out of date. So zero the cached wal-index header to ensure that - ** next time the pager opens a snapshot on this database it knows that - ** the cache needs to be reset. - */ - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - } + u32 iZero; + volatile ht_slot *aHash; - /* Release the locks. */ - walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); - pWal->ckptLock = 0; - WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); + aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; + if( iHash==0 ){ + aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + iZero = 0; + }else{ + iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + } + + *paPgno = &aPgno[-1]; + *paHash = aHash; + *piZero = iZero; + } return rc; } -/* Return the value to pass to a sqlite3_wal_hook callback, the -** number of frames in the WAL at the point of the last commit since -** sqlite3WalCallback() was called. If no commits have occurred since -** the last call, then return 0. +/* +** Return the number of the wal-index page that contains the hash-table +** and page-number array that contain entries corresponding to WAL frame +** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages +** are numbered starting from 0. */ -SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){ - u32 ret = 0; - if( pWal ){ - ret = pWal->iCallback; - pWal->iCallback = 0; +static int walFramePage(u32 iFrame){ + int iHash = (iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1) / HASHTABLE_NPAGE; + assert( (iHash==0 || iFrame>HASHTABLE_NPAGE_ONE) + && (iHash>=1 || iFrame<=HASHTABLE_NPAGE_ONE) + && (iHash<=1 || iFrame>(HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE)) + && (iHash>=2 || iFrame<=HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE) + && (iHash<=2 || iFrame>(HASHTABLE_NPAGE_ONE+2*HASHTABLE_NPAGE)) + ); + return iHash; +} + +/* +** Return the page number associated with frame iFrame in this WAL. +*/ +static u32 walFramePgno(Wal *pWal, u32 iFrame){ + int iHash = walFramePage(iFrame); + if( iHash==0 ){ + return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; } - return (int)ret; + return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; } /* -** This function is called to change the WAL subsystem into or out -** of locking_mode=EXCLUSIVE. -** -** If op is zero, then attempt to change from locking_mode=EXCLUSIVE -** into locking_mode=NORMAL. This means that we must acquire a lock -** on the pWal->readLock byte. If the WAL is already in locking_mode=NORMAL -** or if the acquisition of the lock fails, then return 0. If the -** transition out of exclusive-mode is successful, return 1. This -** operation must occur while the pager is still holding the exclusive -** lock on the main database file. +** Remove entries from the hash table that point to WAL slots greater +** than pWal->hdr.mxFrame. ** -** If op is one, then change from locking_mode=NORMAL into -** locking_mode=EXCLUSIVE. This means that the pWal->readLock must -** be released. Return 1 if the transition is made and 0 if the -** WAL is already in exclusive-locking mode - meaning that this -** routine is a no-op. The pager must already hold the exclusive lock -** on the main database file before invoking this operation. +** This function is called whenever pWal->hdr.mxFrame is decreased due +** to a rollback or savepoint. ** -** If op is negative, then do a dry-run of the op==1 case but do -** not actually change anything. The pager uses this to see if it -** should acquire the database exclusive lock prior to invoking -** the op==1 case. +** At most only the hash table containing pWal->hdr.mxFrame needs to be +** updated. Any later hash tables will be automatically cleared when +** pWal->hdr.mxFrame advances to the point where those hash tables are +** actually needed. */ -SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ - int rc; - assert( pWal->writeLock==0 ); - assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); +static void walCleanupHash(Wal *pWal){ + volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ + volatile u32 *aPgno = 0; /* Page number array for hash table */ + u32 iZero = 0; /* frame == (aHash[x]+iZero) */ + int iLimit = 0; /* Zero values greater than this */ + int nByte; /* Number of bytes to zero in aPgno[] */ + int i; /* Used to iterate through aHash[] */ - /* pWal->readLock is usually set, but might be -1 if there was a - ** prior error while attempting to acquire are read-lock. This cannot - ** happen if the connection is actually in exclusive mode (as no xShmLock - ** locks are taken in this case). Nor should the pager attempt to - ** upgrade to exclusive-mode following such an error. + assert( pWal->writeLock ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE+1 ); + + if( pWal->hdr.mxFrame==0 ) return; + + /* Obtain pointers to the hash-table and page-number array containing + ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed + ** that the page said hash-table and array reside on is already mapped. */ - assert( pWal->readLock>=0 || pWal->lockError ); - assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); + assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); + assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); + walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); - if( op==0 ){ - if( pWal->exclusiveMode ){ - pWal->exclusiveMode = 0; - if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ - pWal->exclusiveMode = 1; - } - rc = pWal->exclusiveMode==0; - }else{ - /* Already in locking_mode=NORMAL */ - rc = 0; + /* Zero all hash-table entries that correspond to frame numbers greater + ** than pWal->hdr.mxFrame. + */ + iLimit = pWal->hdr.mxFrame - iZero; + assert( iLimit>0 ); + for(i=0; iiLimit ){ + aHash[i] = 0; } - }else if( op>0 ){ - assert( pWal->exclusiveMode==0 ); - assert( pWal->readLock>=0 ); - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->exclusiveMode = 1; - rc = 1; - }else{ - rc = pWal->exclusiveMode==0; } - return rc; -} + + /* Zero the entries in the aPgno array that correspond to frames with + ** frame numbers greater than pWal->hdr.mxFrame. + */ + nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); + memset((void *)&aPgno[iLimit+1], 0, nByte); -/* -** Return true if the argument is non-NULL and the WAL module is using -** heap-memory for the wal-index. Otherwise, if the argument is NULL or the -** WAL module is using shared-memory, return false. -*/ -SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ - return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* Verify that the every entry in the mapping region is still reachable + ** via the hash table even after the cleanup. + */ + if( iLimit ){ + int i; /* Loop counter */ + int iKey; /* Hash key */ + for(i=1; i<=iLimit; i++){ + for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ + if( aHash[iKey]==i ) break; + } + assert( aHash[iKey]==i ); + } + } +#endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ } -#endif /* #ifndef SQLITE_OMIT_WAL */ -/************** End of wal.c *************************************************/ -/************** Begin file btmutex.c *****************************************/ /* -** 2007 August 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains code used to implement mutexes on Btree objects. -** This code really belongs in btree.c. But btree.c is getting too -** big and we want to break it down some. This packaged seemed like -** a good breakout. +** Set an entry in the wal-index that will map database page number +** pPage into WAL frame iFrame. */ -/************** Include btreeInt.h in the middle of btmutex.c ****************/ -/************** Begin file btreeInt.h ****************************************/ -/* -** 2004 April 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file implements a external (disk-based) database using BTrees. -** For a detailed discussion of BTrees, refer to -** -** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: -** "Sorting And Searching", pages 473-480. Addison-Wesley -** Publishing Company, Reading, Massachusetts. -** -** The basic idea is that each page of the file contains N database -** entries and N+1 pointers to subpages. -** -** ---------------------------------------------------------------- -** | Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) | -** ---------------------------------------------------------------- -** -** All of the keys on the page that Ptr(0) points to have values less -** than Key(0). All of the keys on page Ptr(1) and its subpages have -** values greater than Key(0) and less than Key(1). All of the keys -** on Ptr(N) and its subpages have values greater than Key(N-1). And -** so forth. -** -** Finding a particular key requires reading O(log(M)) pages from the -** disk where M is the number of entries in the tree. -** -** In this implementation, a single file can hold one or more separate -** BTrees. Each BTree is identified by the index of its root page. The -** key and data for any entry are combined to form the "payload". A -** fixed amount of payload can be carried directly on the database -** page. If the payload is larger than the preset amount then surplus -** bytes are stored on overflow pages. The payload for an entry -** and the preceding pointer are combined to form a "Cell". Each -** page has a small header which contains the Ptr(N) pointer and other -** information such as the size of key and data. -** -** FORMAT DETAILS -** -** The file is divided into pages. The first page is called page 1, -** the second is page 2, and so forth. A page number of zero indicates -** "no such page". The page size can be any power of 2 between 512 and 65536. -** Each page can be either a btree page, a freelist page, an overflow -** page, or a pointer-map page. -** -** The first page is always a btree page. The first 100 bytes of the first -** page contain a special header (the "file header") that describes the file. -** The format of the file header is as follows: -** -** OFFSET SIZE DESCRIPTION -** 0 16 Header string: "SQLite format 3\000" -** 16 2 Page size in bytes. -** 18 1 File format write version -** 19 1 File format read version -** 20 1 Bytes of unused space at the end of each page -** 21 1 Max embedded payload fraction -** 22 1 Min embedded payload fraction -** 23 1 Min leaf payload fraction -** 24 4 File change counter -** 28 4 Reserved for future use -** 32 4 First freelist page -** 36 4 Number of freelist pages in the file -** 40 60 15 4-byte meta values passed to higher layers -** -** 40 4 Schema cookie -** 44 4 File format of schema layer -** 48 4 Size of page cache -** 52 4 Largest root-page (auto/incr_vacuum) -** 56 4 1=UTF-8 2=UTF16le 3=UTF16be -** 60 4 User version -** 64 4 Incremental vacuum mode -** 68 4 unused -** 72 4 unused -** 76 4 unused -** -** All of the integer values are big-endian (most significant byte first). -** -** The file change counter is incremented when the database is changed -** This counter allows other processes to know when the file has changed -** and thus when they need to flush their cache. -** -** The max embedded payload fraction is the amount of the total usable -** space in a page that can be consumed by a single cell for standard -** B-tree (non-LEAFDATA) tables. A value of 255 means 100%. The default -** is to limit the maximum cell size so that at least 4 cells will fit -** on one page. Thus the default max embedded payload fraction is 64. -** -** If the payload for a cell is larger than the max payload, then extra -** payload is spilled to overflow pages. Once an overflow page is allocated, -** as many bytes as possible are moved into the overflow pages without letting -** the cell size drop below the min embedded payload fraction. -** -** The min leaf payload fraction is like the min embedded payload fraction -** except that it applies to leaf nodes in a LEAFDATA tree. The maximum -** payload fraction for a LEAFDATA tree is always 100% (or 255) and it -** not specified in the header. -** -** Each btree pages is divided into three sections: The header, the -** cell pointer array, and the cell content area. Page 1 also has a 100-byte -** file header that occurs before the page header. -** -** |----------------| -** | file header | 100 bytes. Page 1 only. -** |----------------| -** | page header | 8 bytes for leaves. 12 bytes for interior nodes -** |----------------| -** | cell pointer | | 2 bytes per cell. Sorted order. -** | array | | Grows downward -** | | v -** |----------------| -** | unallocated | -** | space | -** |----------------| ^ Grows upwards -** | cell content | | Arbitrary order interspersed with freeblocks. -** | area | | and free space fragments. -** |----------------| -** -** The page headers looks like this: -** -** OFFSET SIZE DESCRIPTION -** 0 1 Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf -** 1 2 byte offset to the first freeblock -** 3 2 number of cells on this page -** 5 2 first byte of the cell content area -** 7 1 number of fragmented free bytes -** 8 4 Right child (the Ptr(N) value). Omitted on leaves. -** -** The flags define the format of this btree page. The leaf flag means that -** this page has no children. The zerodata flag means that this page carries -** only keys and no data. The intkey flag means that the key is a integer -** which is stored in the key size entry of the cell header rather than in -** the payload area. -** -** The cell pointer array begins on the first byte after the page header. -** The cell pointer array contains zero or more 2-byte numbers which are -** offsets from the beginning of the page to the cell content in the cell -** content area. The cell pointers occur in sorted order. The system strives -** to keep free space after the last cell pointer so that new cells can -** be easily added without having to defragment the page. -** -** Cell content is stored at the very end of the page and grows toward the -** beginning of the page. -** -** Unused space within the cell content area is collected into a linked list of -** freeblocks. Each freeblock is at least 4 bytes in size. The byte offset -** to the first freeblock is given in the header. Freeblocks occur in -** increasing order. Because a freeblock must be at least 4 bytes in size, -** any group of 3 or fewer unused bytes in the cell content area cannot -** exist on the freeblock chain. A group of 3 or fewer free bytes is called -** a fragment. The total number of bytes in all fragments is recorded. -** in the page header at offset 7. -** -** SIZE DESCRIPTION -** 2 Byte offset of the next freeblock -** 2 Bytes in this freeblock -** -** Cells are of variable length. Cells are stored in the cell content area at -** the end of the page. Pointers to the cells are in the cell pointer array -** that immediately follows the page header. Cells is not necessarily -** contiguous or in order, but cell pointers are contiguous and in order. -** -** Cell content makes use of variable length integers. A variable -** length integer is 1 to 9 bytes where the lower 7 bits of each -** byte are used. The integer consists of all bytes that have bit 8 set and -** the first byte with bit 8 clear. The most significant byte of the integer -** appears first. A variable-length integer may not be more than 9 bytes long. -** As a special case, all 8 bytes of the 9th byte are used as data. This -** allows a 64-bit integer to be encoded in 9 bytes. -** -** 0x00 becomes 0x00000000 -** 0x7f becomes 0x0000007f -** 0x81 0x00 becomes 0x00000080 -** 0x82 0x00 becomes 0x00000100 -** 0x80 0x7f becomes 0x0000007f -** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 -** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 -** -** Variable length integers are used for rowids and to hold the number of -** bytes of key and data in a btree cell. -** -** The content of a cell looks like this: -** -** SIZE DESCRIPTION -** 4 Page number of the left child. Omitted if leaf flag is set. -** var Number of bytes of data. Omitted if the zerodata flag is set. -** var Number of bytes of key. Or the key itself if intkey flag is set. -** * Payload -** 4 First page of the overflow chain. Omitted if no overflow -** -** Overflow pages form a linked list. Each page except the last is completely -** filled with data (pagesize - 4 bytes). The last page can have as little -** as 1 byte of data. -** -** SIZE DESCRIPTION -** 4 Page number of next overflow page -** * Data -** -** Freelist pages come in two subtypes: trunk pages and leaf pages. The -** file header points to the first in a linked list of trunk page. Each trunk -** page points to multiple leaf pages. The content of a leaf page is -** unspecified. A trunk page looks like this: -** -** SIZE DESCRIPTION -** 4 Page number of next trunk page -** 4 Number of leaf pointers on this page -** * zero or more pages numbers of leaves -*/ - +static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ + int rc; /* Return code */ + u32 iZero = 0; /* One less than frame number of aPgno[1] */ + volatile u32 *aPgno = 0; /* Page number array */ + volatile ht_slot *aHash = 0; /* Hash table */ -/* The following value is the maximum cell size assuming a maximum page -** size give above. -*/ -#define MX_CELL_SIZE(pBt) (pBt->pageSize-8) + rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); -/* The maximum number of cells on a single page of the database. This -** assumes a minimum cell size of 6 bytes (4 bytes for the cell itself -** plus 2 bytes for the index to the cell in the page header). Such -** small cells will be rare, but they are possible. -*/ -#define MX_CELL(pBt) ((pBt->pageSize-8)/6) + /* Assuming the wal-index file was successfully mapped, populate the + ** page number array and hash table entry. + */ + if( rc==SQLITE_OK ){ + int iKey; /* Hash table key */ + int idx; /* Value to write to hash-table slot */ + int nCollide; /* Number of hash collisions */ -/* Forward declarations */ -typedef struct MemPage MemPage; -typedef struct BtLock BtLock; + idx = iFrame - iZero; + assert( idx <= HASHTABLE_NSLOT/2 + 1 ); + + /* If this is the first entry to be added to this hash-table, zero the + ** entire hash table and aPgno[] array before proceding. + */ + if( idx==1 ){ + int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); + memset((void*)&aPgno[1], 0, nByte); + } -/* -** This is a magic string that appears at the beginning of every -** SQLite database in order to identify the file as a real database. -** -** You can change this value at compile-time by specifying a -** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The -** header must be exactly 16 bytes including the zero-terminator so -** the string itself should be 15 characters long. If you change -** the header, then your custom library will not be able to read -** databases generated by the standard tools and the standard tools -** will not be able to read databases created by your custom library. -*/ -#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ -# define SQLITE_FILE_HEADER "SQLite format 3" -#endif + /* If the entry in aPgno[] is already set, then the previous writer + ** must have exited unexpectedly in the middle of a transaction (after + ** writing one or more dirty pages to the WAL to free up memory). + ** Remove the remnants of that writers uncommitted transaction from + ** the hash-table before writing any new entries. + */ + if( aPgno[idx] ){ + walCleanupHash(pWal); + assert( !aPgno[idx] ); + } -/* -** Page type flags. An ORed combination of these flags appear as the -** first byte of on-disk image of every BTree page. -*/ -#define PTF_INTKEY 0x01 -#define PTF_ZERODATA 0x02 -#define PTF_LEAFDATA 0x04 -#define PTF_LEAF 0x08 + /* Write the aPgno[] array entry and the hash-table slot. */ + nCollide = idx; + for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ + if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; + } + aPgno[idx] = iPage; + aHash[iKey] = (ht_slot)idx; -/* -** As each page of the file is loaded into memory, an instance of the following -** structure is appended and initialized to zero. This structure stores -** information about the page that is decoded from the raw file page. -** -** The pParent field points back to the parent page. This allows us to -** walk up the BTree from any leaf to the root. Care must be taken to -** unref() the parent page pointer when this page is no longer referenced. -** The pageDestructor() routine handles that chore. -** -** Access to all fields of this structure is controlled by the mutex -** stored in MemPage.pBt->mutex. -*/ -struct MemPage { - u8 isInit; /* True if previously initialized. MUST BE FIRST! */ - u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ - u8 intKey; /* True if intkey flag is set */ - u8 leaf; /* True if leaf flag is set */ - u8 hasData; /* True if this page stores data */ - u8 hdrOffset; /* 100 for page 1. 0 otherwise */ - u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ - u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ - u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ - u16 cellOffset; /* Index in aData of first cell pointer */ - u16 nFree; /* Number of free bytes on the page */ - u16 nCell; /* Number of cells on this page, local and ovfl */ - u16 maskPage; /* Mask for page offset */ - struct _OvflCell { /* Cells that will not fit on aData[] */ - u8 *pCell; /* Pointers to the body of the overflow cell */ - u16 idx; /* Insert this cell before idx-th non-overflow cell */ - } aOvfl[5]; - BtShared *pBt; /* Pointer to BtShared that this page is part of */ - u8 *aData; /* Pointer to disk image of the page data */ - DbPage *pDbPage; /* Pager page handle */ - Pgno pgno; /* Page number for this page */ -}; +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* Verify that the number of entries in the hash table exactly equals + ** the number of entries in the mapping region. + */ + { + int i; /* Loop counter */ + int nEntry = 0; /* Number of entries in the hash table */ + for(i=0; i0. -** The pSchema field may be set once under BtShared.mutex and -** thereafter is unchanged as long as nRef>0. -** -** isPending: -** -** If a BtShared client fails to obtain a write-lock on a database -** table (because there exists one or more read-locks on the table), -** the shared-cache enters 'pending-lock' state and isPending is -** set to true. -** -** The shared-cache leaves the 'pending lock' state when either of -** the following occur: -** -** 1) The current writer (BtShared.pWriter) concludes its transaction, OR -** 2) The number of locks held by other connections drops to zero. -** -** while in the 'pending-lock' state, no connection may start a new -** transaction. -** -** This feature is included to help prevent writer-starvation. -*/ -struct BtShared { - Pager *pPager; /* The page cache */ - sqlite3 *db; /* Database connection currently using this Btree */ - BtCursor *pCursor; /* A list of all open cursors */ - MemPage *pPage1; /* First page of the database */ - u8 readOnly; /* True if the underlying file is readonly */ - u8 pageSizeFixed; /* True if the page size can no longer be changed */ - u8 secureDelete; /* True if secure_delete is enabled */ - u8 initiallyEmpty; /* Database is empty at start of transaction */ - u8 openFlags; /* Flags to sqlite3BtreeOpen() */ -#ifndef SQLITE_OMIT_AUTOVACUUM - u8 autoVacuum; /* True if auto-vacuum is enabled */ - u8 incrVacuum; /* True if incr-vacuum is enabled */ -#endif - u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ - u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ - u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ - u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ - u8 inTransaction; /* Transaction state */ - u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ - u32 pageSize; /* Total number of bytes on a page */ - u32 usableSize; /* Number of usable bytes on each page */ - int nTransaction; /* Number of open transactions (read + write) */ - u32 nPage; /* Number of pages in the database */ - void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ - void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ - sqlite3_mutex *mutex; /* Non-recursive mutex required to access this struct */ - Bitvec *pHasContent; /* Set of pages moved to free-list this transaction */ -#ifndef SQLITE_OMIT_SHARED_CACHE - int nRef; /* Number of references to this structure */ - BtShared *pNext; /* Next on a list of sharable BtShared structs */ - BtLock *pLock; /* List of locks held on this shared-btree struct */ - Btree *pWriter; /* Btree with currently open write transaction */ - u8 isExclusive; /* True if pWriter has an EXCLUSIVE lock on the db */ - u8 isPending; /* If waiting for read-locks to clear */ -#endif - u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ -}; + /* Obtain an exclusive lock on all byte in the locking range not already + ** locked by the caller. The caller is guaranteed to have locked the + ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte. + ** If successful, the same bytes that are locked here are unlocked before + ** this function returns. + */ + assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); + assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); + assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); + assert( pWal->writeLock ); + iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; + nLock = SQLITE_SHM_NLOCK - iLock; + rc = walLockExclusive(pWal, iLock, nLock); + if( rc ){ + return rc; + } + WALTRACE(("WAL%p: recovery begin...\n", pWal)); -/* -** An instance of the following structure is used to hold information -** about a cell. The parseCellPtr() function fills in this structure -** based on information extract from the raw disk page. -*/ -typedef struct CellInfo CellInfo; -struct CellInfo { - u8 *pCell; /* Pointer to the start of cell content */ - i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ - u32 nData; /* Number of bytes of data */ - u32 nPayload; /* Total amount of payload */ - u16 nHeader; /* Size of the cell content header in bytes */ - u16 nLocal; /* Amount of payload held locally */ - u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ - u16 nSize; /* Size of the cell content on the main b-tree page */ -}; + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); -/* -** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than -** this will be declared corrupt. This value is calculated based on a -** maximum database size of 2^31 pages a minimum fanout of 2 for a -** root-node and 3 for all other internal nodes. -** -** If a tree that appears to be taller than this is encountered, it is -** assumed that the database is corrupt. -*/ -#define BTCURSOR_MAX_DEPTH 20 + rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } -/* -** A cursor is a pointer to a particular entry within a particular -** b-tree within a database file. -** -** The entry is identified by its MemPage and the index in -** MemPage.aCell[] of the entry. -** -** A single database file can shared by two more database connections, -** but cursors cannot be shared. Each cursor is associated with a -** particular database connection identified BtCursor.pBtree.db. -** -** Fields in this structure are accessed under the BtShared.mutex -** found at self->pBt->mutex. -*/ -struct BtCursor { - Btree *pBtree; /* The Btree to which this cursor belongs */ - BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ - struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ - Pgno pgnoRoot; /* The root page of this tree */ - sqlite3_int64 cachedRowid; /* Next rowid cache. 0 means not valid */ - CellInfo info; /* A parse of the cell we are pointing at */ - u8 wrFlag; /* True if writable */ - u8 atLast; /* Cursor pointing to the last entry */ - u8 validNKey; /* True if info.nKey is valid */ - u8 eState; /* One of the CURSOR_XXX constants (see below) */ - void *pKey; /* Saved key that was cursor's last known position */ - i64 nKey; /* Size of pKey, or last integer key */ - int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ -#ifndef SQLITE_OMIT_INCRBLOB - u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ - Pgno *aOverflow; /* Cache of overflow page locations */ -#endif - i16 iPage; /* Index of current page in apPage */ - MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ - u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ -}; + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int iFrame; /* Index of last frame read */ + i64 iOffset; /* Next offset to read from log file */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ -/* -** Potential values for BtCursor.eState. -** -** CURSOR_VALID: -** Cursor points to a valid entry. getPayload() etc. may be called. -** -** CURSOR_INVALID: -** Cursor does not point to a valid entry. This can happen (for example) -** because the table is empty or because BtreeCursorFirst() has not been -** called. -** -** CURSOR_REQUIRESEEK: -** The table that this cursor was opened on still exists, but has been -** modified since the cursor was last used. The cursor position is saved -** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in -** this state, restoreCursorPosition() can be called to attempt to -** seek the cursor to the saved position. -** -** CURSOR_FAULT: -** A unrecoverable error (an I/O error or a malloc failure) has occurred -** on a different connection that shares the BtShared cache with this -** cursor. The error has left the cache in an inconsistent state. -** Do nothing else with this cursor. Any attempt to use the cursor -** should return the error code stored in BtCursor.skip -*/ -#define CURSOR_INVALID 0 -#define CURSOR_VALID 1 -#define CURSOR_REQUIRESEEK 2 -#define CURSOR_FAULT 3 + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } -/* -** The database page the PENDING_BYTE occupies. This page is never used. -*/ -# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + goto finished; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); -/* -** These macros define the location of the pointer-map entry for a -** database page. The first argument to each is the number of usable -** bytes on each page of the database (often 1024). The second is the -** page number to look up in the pointer map. -** -** PTRMAP_PAGENO returns the database page number of the pointer-map -** page that stores the required pointer. PTRMAP_PTROFFSET returns -** the offset of the requested map entry. -** -** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page, -** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be -** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements -** this test. -*/ -#define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno) -#define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1)) -#define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno)) + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + goto finished; + } -/* -** The pointer map is a lookup table that identifies the parent page for -** each child page in the database file. The parent page is the page that -** contains a pointer to the child. Every page in the database contains -** 0 or 1 parent pages. (In this context 'database page' refers -** to any page that is not part of the pointer map itself.) Each pointer map -** entry consists of a single byte 'type' and a 4 byte parent page number. -** The PTRMAP_XXX identifiers below are the valid types. -** -** The purpose of the pointer map is to facility moving pages from one -** position in the file to another as part of autovacuum. When a page -** is moved, the pointer in its parent must be updated to point to the -** new location. The pointer map is used to locate the parent page quickly. -** -** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not -** used in this case. -** -** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number -** is not used in this case. -** -** PTRMAP_OVERFLOW1: The database page is the first page in a list of -** overflow pages. The page number identifies the page that -** contains the cell with a pointer to this overflow page. -** -** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of -** overflow pages. The page-number identifies the previous -** page in the overflow page list. -** -** PTRMAP_BTREE: The database page is a non-root btree page. The page number -** identifies the parent page in the btree. -*/ -#define PTRMAP_ROOTPAGE 1 -#define PTRMAP_FREEPAGE 2 -#define PTRMAP_OVERFLOW1 3 -#define PTRMAP_OVERFLOW2 4 -#define PTRMAP_BTREE 5 + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; + goto finished; + } -/* A bunch of assert() statements to check the transaction state variables -** of handle p (type Btree*) are internally consistent. -*/ -#define btreeIntegrity(p) \ - assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ - assert( p->pBt->inTransaction>=p->inTrans ); + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc(szFrame); + if( !aFrame ){ + rc = SQLITE_NOMEM; + goto recovery_error; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + /* Read all frames from the log file. */ + iFrame = 0; + for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + int isValid; /* True if this frame is valid */ -/* -** The ISAUTOVACUUM macro is used within balance_nonroot() to determine -** if the database supports auto-vacuum or not. Because it is used -** within an expression that is an argument to another macro -** (sqliteMallocRaw), it is not possible to use conditional compilation. -** So, this macro is defined instead. -*/ -#ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) -#else -#define ISAUTOVACUUM 0 -#endif + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, ++iFrame, pgno); + if( rc!=SQLITE_OK ) break; + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } -/* -** This structure is passed around through all the sanity checking routines -** in order to keep track of some global state information. -*/ -typedef struct IntegrityCk IntegrityCk; -struct IntegrityCk { - BtShared *pBt; /* The tree being checked out */ - Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ - Pgno nPage; /* Number of pages in the database */ - int *anRef; /* Number of times each page is referenced */ - int mxErr; /* Stop accumulating errors when this reaches zero */ - int nErr; /* Number of messages written to zErrMsg so far */ - int mallocFailed; /* A memory allocation error has occurred */ - StrAccum errMsg; /* Accumulate the error message text here */ -}; + sqlite3_free(aFrame); + } -/* -** Read or write a two- and four-byte big-endian integer values. -*/ -#define get2byte(x) ((x)[0]<<8 | (x)[1]) -#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) -#define get4byte sqlite3Get4byte -#define put4byte sqlite3Put4byte +finished: + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo; + int i; + pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + walIndexWriteHdr(pWal); -/************** End of btreeInt.h ********************************************/ -/************** Continuing where we left off in btmutex.c ********************/ -#ifndef SQLITE_OMIT_SHARED_CACHE -#if SQLITE_THREADSAFE + /* Reset the checkpoint-header. This is safe because this thread is + ** currently holding locks that exclude all other readers, writers and + ** checkpointers. + */ + pInfo = walCkptInfo(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[0] = 0; + for(i=1; iaReadMark[i] = READMARK_NOT_USED; -/* -** Obtain the BtShared mutex associated with B-Tree handle p. Also, -** set BtShared.db to the database handle associated with p and the -** p->locked boolean to true. -*/ -static void lockBtreeMutex(Btree *p){ - assert( p->locked==0 ); - assert( sqlite3_mutex_notheld(p->pBt->mutex) ); - assert( sqlite3_mutex_held(p->db->mutex) ); + /* If more than one frame was recovered from the log file, report an + ** event via sqlite3_log(). This is to help with identifying performance + ** problems caused by applications routinely shutting down without + ** checkpointing the log file. + */ + if( pWal->hdr.nPage ){ + sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", + pWal->hdr.nPage, pWal->zWalName + ); + } + } - sqlite3_mutex_enter(p->pBt->mutex); - p->pBt->db = p->db; - p->locked = 1; +recovery_error: + WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); + walUnlockExclusive(pWal, iLock, nLock); + return rc; } /* -** Release the BtShared mutex associated with B-Tree handle p and -** clear the p->locked boolean. +** Close an open wal-index. */ -static void unlockBtreeMutex(Btree *p){ - assert( p->locked==1 ); - assert( sqlite3_mutex_held(p->pBt->mutex) ); - assert( sqlite3_mutex_held(p->db->mutex) ); - assert( p->db==p->pBt->db ); - - sqlite3_mutex_leave(p->pBt->mutex); - p->locked = 0; +static void walIndexClose(Wal *pWal, int isDelete){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void *)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + }else{ + sqlite3OsShmUnmap(pWal->pDbFd, isDelete); + } } -/* -** Enter a mutex on the given BTree object. +/* +** Open a connection to the WAL file zWalName. The database file must +** already be opened on connection pDbFd. The buffer that zWalName points +** to must remain valid for the lifetime of the returned Wal* handle. ** -** If the object is not sharable, then no mutex is ever required -** and this routine is a no-op. The underlying mutex is non-recursive. -** But we keep a reference count in Btree.wantToLock so the behavior -** of this interface is recursive. +** A SHARED lock should be held on the database file when this function +** is called. The purpose of this SHARED lock is to prevent any other +** client from unlinking the WAL or wal-index file. If another process +** were to do this just after this client opened one of these files, the +** system would be badly broken. ** -** To avoid deadlocks, multiple Btrees are locked in the same order -** by all database connections. The p->pNext is a list of other -** Btrees belonging to the same database connection as the p Btree -** which need to be locked after p. If we cannot get a lock on -** p, then first unlock all of the others on p->pNext, then wait -** for the lock to become available on p, then relock all of the -** subsequent Btrees that desire a lock. +** If the log file is successfully opened, SQLITE_OK is returned and +** *ppWal is set to point to a new WAL handle. If an error occurs, +** an SQLite error code is returned and *ppWal is left unmodified. */ -SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ - Btree *pLater; +SQLITE_PRIVATE int sqlite3WalOpen( + sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + sqlite3_file *pDbFd, /* The open database file */ + const char *zWalName, /* Name of the WAL file */ + int bNoShm, /* True to run in heap-memory mode */ + i64 mxWalSize, /* Truncate WAL to this size on reset */ + Wal **ppWal /* OUT: Allocated Wal handle */ +){ + int rc; /* Return Code */ + Wal *pRet; /* Object to allocate and return */ + int flags; /* Flags passed to OsOpen() */ - /* Some basic sanity checking on the Btree. The list of Btrees - ** connected by pNext and pPrev should be in sorted order by - ** Btree.pBt value. All elements of the list should belong to - ** the same connection. Only shared Btrees are on the list. */ - assert( p->pNext==0 || p->pNext->pBt>p->pBt ); - assert( p->pPrev==0 || p->pPrev->pBtpBt ); - assert( p->pNext==0 || p->pNext->db==p->db ); - assert( p->pPrev==0 || p->pPrev->db==p->db ); - assert( p->sharable || (p->pNext==0 && p->pPrev==0) ); + assert( zWalName && zWalName[0] ); + assert( pDbFd ); - /* Check for locking consistency */ - assert( !p->locked || p->wantToLock>0 ); - assert( p->sharable || p->wantToLock==0 ); + /* In the amalgamation, the os_unix.c and os_win.c source files come before + ** this source file. Verify that the #defines of the locking byte offsets + ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. + */ +#ifdef WIN_SHM_BASE + assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); +#endif +#ifdef UNIX_SHM_BASE + assert( UNIX_SHM_BASE==WALINDEX_LOCK_OFFSET ); +#endif - /* We should already hold a lock on the database connection */ - assert( sqlite3_mutex_held(p->db->mutex) ); - /* Unless the database is sharable and unlocked, then BtShared.db - ** should already be set correctly. */ - assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db ); + /* Allocate an instance of struct Wal to return. */ + *ppWal = 0; + pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); + if( !pRet ){ + return SQLITE_NOMEM; + } - if( !p->sharable ) return; - p->wantToLock++; - if( p->locked ) return; + pRet->pVfs = pVfs; + pRet->pWalFd = (sqlite3_file *)&pRet[1]; + pRet->pDbFd = pDbFd; + pRet->readLock = -1; + pRet->mxWalSize = mxWalSize; + pRet->zWalName = zWalName; + pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); - /* In most cases, we should be able to acquire the lock we - ** want without having to go throught the ascending lock - ** procedure that follows. Just be sure not to block. - */ - if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ - p->pBt->db = p->db; - p->locked = 1; - return; + /* Open file handle on the write-ahead log file. */ + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ + pRet->readOnly = WAL_RDONLY; } - /* To avoid deadlock, first release all locks with a larger - ** BtShared address. Then acquire our lock. Then reacquire - ** the other BtShared locks that we used to hold in ascending - ** order. - */ - for(pLater=p->pNext; pLater; pLater=pLater->pNext){ - assert( pLater->sharable ); - assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt ); - assert( !pLater->locked || pLater->wantToLock>0 ); - if( pLater->locked ){ - unlockBtreeMutex(pLater); - } - } - lockBtreeMutex(p); - for(pLater=p->pNext; pLater; pLater=pLater->pNext){ - if( pLater->wantToLock ){ - lockBtreeMutex(pLater); - } + if( rc!=SQLITE_OK ){ + walIndexClose(pRet, 0); + sqlite3OsClose(pRet->pWalFd); + sqlite3_free(pRet); + }else{ + *ppWal = pRet; + WALTRACE(("WAL%d: opened\n", pRet)); } + return rc; } /* -** Exit the recursive mutex on a Btree. +** Change the size to which the WAL file is trucated on each reset. */ -SQLITE_PRIVATE void sqlite3BtreeLeave(Btree *p){ - if( p->sharable ){ - assert( p->wantToLock>0 ); - p->wantToLock--; - if( p->wantToLock==0 ){ - unlockBtreeMutex(p); - } - } +SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ + if( pWal ) pWal->mxWalSize = iLimit; } -#ifndef NDEBUG /* -** Return true if the BtShared mutex is held on the btree, or if the -** B-Tree is not marked as sharable. +** Find the smallest page number out of all pages held in the WAL that +** has not been returned by any prior invocation of this method on the +** same WalIterator object. Write into *piFrame the frame index where +** that page was last written into the WAL. Write into *piPage the page +** number. ** -** This routine is used only from within assert() statements. +** Return 0 on success. If there are no pages in the WAL with a page +** number larger than *piPage, then return 1. */ -SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ - assert( p->sharable==0 || p->locked==0 || p->wantToLock>0 ); - assert( p->sharable==0 || p->locked==0 || p->db==p->pBt->db ); - assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->pBt->mutex) ); - assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->db->mutex) ); - - return (p->sharable==0 || p->locked); -} -#endif +static int walIteratorNext( + WalIterator *p, /* Iterator */ + u32 *piPage, /* OUT: The page number of the next page */ + u32 *piFrame /* OUT: Wal frame index of next page */ +){ + u32 iMin; /* Result pgno must be greater than iMin */ + u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ + int i; /* For looping through segments */ + iMin = p->iPrior; + assert( iMin<0xffffffff ); + for(i=p->nSegment-1; i>=0; i--){ + struct WalSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextnEntry ){ + u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgiZero + pSegment->aIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } + } -#ifndef SQLITE_OMIT_INCRBLOB -/* -** Enter and leave a mutex on a Btree given a cursor owned by that -** Btree. These entry points are used by incremental I/O and can be -** omitted if that module is not used. -*/ -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ - sqlite3BtreeEnter(pCur->pBtree); -} -SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ - sqlite3BtreeLeave(pCur->pBtree); + *piPage = p->iPrior = iRet; + return (iRet==0xFFFFFFFF); } -#endif /* SQLITE_OMIT_INCRBLOB */ - /* -** Enter the mutex on every Btree associated with a database -** connection. This is needed (for example) prior to parsing -** a statement since we will be comparing table and column names -** against all schemas and we do not want those schemas being -** reset out from under us. +** This function merges two sorted lists into a single sorted list. ** -** There is a corresponding leave-all procedures. +** aLeft[] and aRight[] are arrays of indices. The sort key is +** aContent[aLeft[]] and aContent[aRight[]]. Upon entry, the following +** is guaranteed for all Jmutex) ); - for(i=0; inDb; i++){ - p = db->aDb[i].pBt; - assert( !p || (p->locked==0 && p->sharable) || p->pBt->db==p->db ); - if( p && p->sharable ){ - p->wantToLock++; - if( !p->locked ){ - assert( p->wantToLock==1 ); - while( p->pPrev ) p = p->pPrev; - /* Reason for ALWAYS: There must be at least on unlocked Btree in - ** the chain. Otherwise the !p->locked test above would have failed */ - while( p->locked && ALWAYS(p->pNext) ) p = p->pNext; - for(pLater = p->pNext; pLater; pLater=pLater->pNext){ - if( pLater->locked ){ - unlockBtreeMutex(pLater); - } - } - while( p ){ - lockBtreeMutex(p); - p = p->pNext; - } - } - } - } -} -SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ - int i; - Btree *p; - assert( sqlite3_mutex_held(db->mutex) ); - for(i=0; inDb; i++){ - p = db->aDb[i].pBt; - if( p && p->sharable ){ - assert( p->wantToLock>0 ); - p->wantToLock--; - if( p->wantToLock==0 ){ - unlockBtreeMutex(p); - } +static void walMerge( + const u32 *aContent, /* Pages in wal - keys for the sort */ + ht_slot *aLeft, /* IN: Left hand input list */ + int nLeft, /* IN: Elements in array *paLeft */ + ht_slot **paRight, /* IN/OUT: Right hand input list */ + int *pnRight, /* IN/OUT: Elements in *paRight */ + ht_slot *aTmp /* Temporary buffer */ +){ + int iLeft = 0; /* Current index in aLeft */ + int iRight = 0; /* Current index in aRight */ + int iOut = 0; /* Current index in output buffer */ + int nRight = *pnRight; + ht_slot *aRight = *paRight; + + assert( nLeft>0 && nRight>0 ); + while( iRight=nRight || aContent[aLeft[iLeft]]=nLeft || aContent[aLeft[iLeft]]>dbpage ); + assert( iRight>=nRight || aContent[aRight[iRight]]>dbpage ); } + + *paRight = aLeft; + *pnRight = iOut; + memcpy(aLeft, aTmp, sizeof(aTmp[0])*iOut); } -#ifndef NDEBUG /* -** Return true if the current thread holds the database connection -** mutex and all required BtShared mutexes. +** Sort the elements in list aList using aContent[] as the sort key. +** Remove elements with duplicate keys, preferring to keep the +** larger aList[] values. ** -** This routine is used inside assert() statements only. +** The aList[] entries are indices into aContent[]. The values in +** aList[] are to be sorted so that for all Jmutex) ){ - return 0; +static void walMergesort( + const u32 *aContent, /* Pages in wal */ + ht_slot *aBuffer, /* Buffer of at least *pnList items to use */ + ht_slot *aList, /* IN/OUT: List to sort */ + int *pnList /* IN/OUT: Number of elements in aList[] */ +){ + struct Sublist { + int nList; /* Number of elements in aList */ + ht_slot *aList; /* Pointer to sub-list content */ + }; + + const int nList = *pnList; /* Size of input list */ + int nMerge = 0; /* Number of elements in list aMerge */ + ht_slot *aMerge = 0; /* List to be merged */ + int iList; /* Index into input list */ + int iSub = 0; /* Index into aSub array */ + struct Sublist aSub[13]; /* Array of sub-lists */ + + memset(aSub, 0, sizeof(aSub)); + assert( nList<=HASHTABLE_NPAGE && nList>0 ); + assert( HASHTABLE_NPAGE==(1<<(ArraySize(aSub)-1)) ); + + for(iList=0; iListaList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); + } + aSub[iSub].aList = aMerge; + aSub[iSub].nList = nMerge; } - for(i=0; inDb; i++){ - Btree *p; - p = db->aDb[i].pBt; - if( p && p->sharable && - (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){ - return 0; + + for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); } } - return 1; -} -#endif /* NDEBUG */ + assert( aMerge==aList ); + *pnList = nMerge; -/* -** Add a new Btree pointer to a BtreeMutexArray. -** if the pointer can possibly be shared with -** another database connection. -** -** The pointers are kept in sorted order by pBtree->pBt. That -** way when we go to enter all the mutexes, we can enter them -** in order without every having to backup and retry and without -** worrying about deadlock. -** -** The number of shared btrees will always be small (usually 0 or 1) -** so an insertion sort is an adequate algorithm here. -*/ -SQLITE_PRIVATE void sqlite3BtreeMutexArrayInsert(BtreeMutexArray *pArray, Btree *pBtree){ - int i, j; - BtShared *pBt; - if( pBtree==0 || pBtree->sharable==0 ) return; -#ifndef NDEBUG +#ifdef SQLITE_DEBUG { - for(i=0; inMutex; i++){ - assert( pArray->aBtree[i]!=pBtree ); + int i; + for(i=1; i<*pnList; i++){ + assert( aContent[aList[i]] > aContent[aList[i-1]] ); } } #endif - assert( pArray->nMutex>=0 ); - assert( pArray->nMutexaBtree)-1 ); - pBt = pBtree->pBt; - for(i=0; inMutex; i++){ - assert( pArray->aBtree[i]!=pBtree ); - if( pArray->aBtree[i]->pBt>pBt ){ - for(j=pArray->nMutex; j>i; j--){ - pArray->aBtree[j] = pArray->aBtree[j-1]; - } - pArray->aBtree[i] = pBtree; - pArray->nMutex++; - return; - } - } - pArray->aBtree[pArray->nMutex++] = pBtree; } -/* -** Enter the mutex of every btree in the array. This routine is -** called at the beginning of sqlite3VdbeExec(). The mutexes are -** exited at the end of the same function. +/* +** Free an iterator allocated by walIteratorInit(). */ -SQLITE_PRIVATE void sqlite3BtreeMutexArrayEnter(BtreeMutexArray *pArray){ - int i; - for(i=0; inMutex; i++){ - Btree *p = pArray->aBtree[i]; - /* Some basic sanity checking */ - assert( i==0 || pArray->aBtree[i-1]->pBtpBt ); - assert( !p->locked || p->wantToLock>0 ); - - /* We should already hold a lock on the database connection */ - assert( sqlite3_mutex_held(p->db->mutex) ); - - /* The Btree is sharable because only sharable Btrees are entered - ** into the array in the first place. */ - assert( p->sharable ); - - p->wantToLock++; - if( !p->locked ){ - lockBtreeMutex(p); - } - } +static void walIteratorFree(WalIterator *p){ + sqlite3ScratchFree(p); } /* -** Leave the mutex of every btree in the group. +** Construct a WalInterator object that can be used to loop over all +** pages in the WAL in ascending order. The caller must hold the checkpoint +** lock. +** +** On success, make *pp point to the newly allocated WalInterator object +** return SQLITE_OK. Otherwise, return an error code. If this routine +** returns an error, the value of *pp is undefined. +** +** The calling routine should invoke walIteratorFree() to destroy the +** WalIterator object when it has finished with it. */ -SQLITE_PRIVATE void sqlite3BtreeMutexArrayLeave(BtreeMutexArray *pArray){ - int i; - for(i=0; inMutex; i++){ - Btree *p = pArray->aBtree[i]; - /* Some basic sanity checking */ - assert( i==0 || pArray->aBtree[i-1]->pBtpBt ); - assert( p->locked ); - assert( p->wantToLock>0 ); +static int walIteratorInit(Wal *pWal, WalIterator **pp){ + WalIterator *p; /* Return value */ + int nSegment; /* Number of segments to merge */ + u32 iLast; /* Last frame in log */ + int nByte; /* Number of bytes to allocate */ + int i; /* Iterator variable */ + ht_slot *aTmp; /* Temp space used by merge-sort */ + int rc = SQLITE_OK; /* Return Code */ - /* We should already hold a lock on the database connection */ - assert( sqlite3_mutex_held(p->db->mutex) ); + /* This routine only runs while holding the checkpoint lock. And + ** it only runs if there is actually content in the log (mxFrame>0). + */ + assert( pWal->ckptLock && pWal->hdr.mxFrame>0 ); + iLast = pWal->hdr.mxFrame; - p->wantToLock--; - if( p->wantToLock==0 ){ - unlockBtreeMutex(p); - } + /* Allocate space for the WalIterator object. */ + nSegment = walFramePage(iLast) + 1; + nByte = sizeof(WalIterator) + + (nSegment-1)*sizeof(struct WalSegment) + + iLast*sizeof(ht_slot); + p = (WalIterator *)sqlite3ScratchMalloc(nByte); + if( !p ){ + return SQLITE_NOMEM; } -} + memset(p, 0, nByte); + p->nSegment = nSegment; -#else -SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ - p->pBt->db = p->db; -} -SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ - int i; - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p ){ - p->pBt->db = p->db; - } + /* Allocate temporary space used by the merge-sort routine. This block + ** of memory will be freed before this function returns. + */ + aTmp = (ht_slot *)sqlite3ScratchMalloc( + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); + if( !aTmp ){ + rc = SQLITE_NOMEM; } -} -#endif /* if SQLITE_THREADSAFE */ -#endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ -/************** End of btmutex.c *********************************************/ -/************** Begin file btree.c *******************************************/ -/* -** 2004 April 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file implements a external (disk-based) database using BTrees. -** See the header comment on "btreeInt.h" for additional information. -** Including a description of file format and an overview of operation. -*/ + for(i=0; rc==SQLITE_OK && iaSegment[p->nSegment])[iZero]; + iZero++; + + for(j=0; jaSegment[i].iZero = iZero; + p->aSegment[i].nEntry = nEntry; + p->aSegment[i].aIndex = aIndex; + p->aSegment[i].aPgno = (u32 *)aPgno; + } + } + sqlite3ScratchFree(aTmp); -/* -** Extract a 2-byte big-endian integer from an array of unsigned bytes. -** But if the value is zero, make it 65536. -** -** This routine is used to extract the "offset to cell content area" value -** from the header of a btree page. If the page size is 65536 and the page -** is empty, the offset should be 65536, but the 2-byte value stores zero. -** This routine makes the necessary adjustment to 65536. -*/ -#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) + if( rc!=SQLITE_OK ){ + walIteratorFree(p); + } + *pp = p; + return rc; +} -#ifndef SQLITE_OMIT_SHARED_CACHE /* -** A list of BtShared objects that are eligible for participation -** in shared cache. This variable has file scope during normal builds, -** but the test harness needs to access it so we make it global for -** test builds. -** -** Access to this variable is protected by SQLITE_MUTEX_STATIC_MASTER. +** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and +** n. If the attempt fails and parameter xBusy is not NULL, then it is a +** busy-handler function. Invoke it and retry the lock until either the +** lock is successfully obtained or the busy-handler returns 0. */ -#ifdef SQLITE_TEST -SQLITE_PRIVATE BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; -#else -static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; -#endif -#endif /* SQLITE_OMIT_SHARED_CACHE */ +static int walBusyLock( + Wal *pWal, /* WAL connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int lockIdx, /* Offset of first byte to lock */ + int n /* Number of bytes to lock */ +){ + int rc; + do { + rc = walLockExclusive(pWal, lockIdx, n); + }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); + return rc; +} -#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Enable or disable the shared pager and schema features. -** -** This routine has no effect on existing database connections. -** The shared cache setting effects only future calls to -** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). +** The cache of the wal-index header must be valid to call this function. +** Return the page-size in bytes used by the database. */ -SQLITE_API int sqlite3_enable_shared_cache(int enable){ - sqlite3GlobalConfig.sharedCacheEnabled = enable; - return SQLITE_OK; +static int walPagesize(Wal *pWal){ + return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } -#endif - - - -#ifdef SQLITE_OMIT_SHARED_CACHE - /* - ** The functions querySharedCacheTableLock(), setSharedCacheTableLock(), - ** and clearAllSharedCacheTableLocks() - ** manipulate entries in the BtShared.pLock linked list used to store - ** shared-cache table level locks. If the library is compiled with the - ** shared-cache feature disabled, then there is only ever one user - ** of each BtShared structure and so this locking is not necessary. - ** So define the lock related functions as no-ops. - */ - #define querySharedCacheTableLock(a,b,c) SQLITE_OK - #define setSharedCacheTableLock(a,b,c) SQLITE_OK - #define clearAllSharedCacheTableLocks(a) - #define downgradeAllSharedCacheTableLocks(a) - #define hasSharedCacheTableLock(a,b,c,d) 1 - #define hasReadConflicts(a, b) 0 -#endif - -#ifndef SQLITE_OMIT_SHARED_CACHE -#ifdef SQLITE_DEBUG /* -**** This function is only used as part of an assert() statement. *** +** Copy as much content as we can from the WAL back into the database file +** in response to an sqlite3_wal_checkpoint() request or the equivalent. ** -** Check to see if pBtree holds the required locks to read or write to the -** table with root page iRoot. Return 1 if it does and 0 if not. +** The amount of information copies from WAL to database might be limited +** by active readers. This routine will never overwrite a database page +** that a concurrent reader might be using. ** -** For example, when writing to a table with root-page iRoot via -** Btree connection pBtree: +** All I/O barrier operations (a.k.a fsyncs) occur in this routine when +** SQLite is in WAL-mode in synchronous=NORMAL. That means that if +** checkpoints are always run by a background thread or background +** process, foreground threads will never block on a lengthy fsync call. ** -** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); +** Fsync is called on the WAL before writing content out of the WAL and +** into the database. This ensures that if the new content is persistent +** in the WAL and can be recovered following a power-loss or hard reset. ** -** When writing to an index that resides in a sharable database, the -** caller should have first obtained a lock specifying the root page of -** the corresponding table. This makes things a bit more complicated, -** as this module treats each table as a separate structure. To determine -** the table corresponding to the index being written, this -** function has to search through the database schema. +** Fsync is also called on the database file if (and only if) the entire +** WAL content is copied into the database file. This second fsync makes +** it safe to delete the WAL since the new content will persist in the +** database file. ** -** Instead of a lock on the table/index rooted at page iRoot, the caller may -** hold a write-lock on the schema table (root page 1). This is also -** acceptable. +** This routine uses and updates the nBackfill field of the wal-index header. +** This is the only routine tha will increase the value of nBackfill. +** (A WAL reset or recovery will revert nBackfill to zero, but not increase +** its value.) +** +** The caller must be holding sufficient locks to ensure that no other +** checkpoint is running (in any other thread or process) at the same +** time. */ -static int hasSharedCacheTableLock( - Btree *pBtree, /* Handle that must hold lock */ - Pgno iRoot, /* Root page of b-tree */ - int isIndex, /* True if iRoot is the root of an index b-tree */ - int eLockType /* Required lock type (READ_LOCK or WRITE_LOCK) */ +static int walCheckpoint( + Wal *pWal, /* Wal connection */ + int eMode, /* One of PASSIVE, FULL or RESTART */ + int (*xBusyCall)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags for OsSync() (or 0) */ + u8 *zBuf /* Temporary buffer to use */ ){ - Schema *pSchema = (Schema *)pBtree->pBt->pSchema; - Pgno iTab = 0; - BtLock *pLock; + int rc; /* Return code */ + int szPage; /* Database page-size */ + WalIterator *pIter = 0; /* Wal iterator context */ + u32 iDbpage = 0; /* Next database page to write */ + u32 iFrame = 0; /* Wal frame containing data for iDbpage */ + u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ + int i; /* Loop counter */ + volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ - /* If this database is not shareable, or if the client is reading - ** and has the read-uncommitted flag set, then no lock is required. - ** Return true immediately. - */ - if( (pBtree->sharable==0) - || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted)) - ){ - return 1; + szPage = walPagesize(pWal); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + pInfo = walCkptInfo(pWal); + if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; + + /* Allocate the iterator */ + rc = walIteratorInit(pWal, &pIter); + if( rc!=SQLITE_OK ){ + return rc; } + assert( pIter ); - /* If the client is reading or writing an index and the schema is - ** not loaded, then it is too difficult to actually check to see if - ** the correct locks are held. So do not bother - just return true. - ** This case does not come up very often anyhow. + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + + /* Compute in mxSafeFrame the index of the last frame of the WAL that is + ** safe to write into the database. Frames beyond mxSafeFrame might + ** overwrite database pages that are in use by active readers and thus + ** cannot be backfilled from the WAL. */ - if( isIndex && (!pSchema || (pSchema->flags&DB_SchemaLoaded)==0) ){ - return 1; + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = READMARK_NOT_USED; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; + } + } } - /* Figure out the root-page that the lock should be held on. For table - ** b-trees, this is just the root page of the b-tree being read or - ** written. For index b-trees, it is the root page of the associated - ** table. */ - if( isIndex ){ - HashElem *p; - for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ - Index *pIdx = (Index *)sqliteHashData(p); - if( pIdx->tnum==(int)iRoot ){ - iTab = pIdx->pTable->tnum; + if( pInfo->nBackfillnBackfill; + + /* Sync the WAL to disk */ + if( sync_flags ){ + rc = sqlite3OsSync(pWal->pWalFd, sync_flags); + } + + /* If the database file may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); } } - }else{ - iTab = iRoot; - } - /* Search for the required lock. Either a write-lock on root-page iTab, a - ** write-lock on the schema table, or (if the client is reading) a - ** read-lock on iTab will suffice. Return 1 if any of these are found. */ - for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ - if( pLock->pBtree==pBtree - && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) - && pLock->eLock>=eLockType - ){ - return 1; + /* Iterate through the contents of the WAL, copying data to the db file. */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + } + + /* If work was actually accomplished... */ + if( rc==SQLITE_OK ){ + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK && sync_flags ){ + rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + } + } + if( rc==SQLITE_OK ){ + pInfo->nBackfill = mxSafeFrame; + } } + + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); } - /* Failed to find the required lock. */ - return 0; -} -#endif /* SQLITE_DEBUG */ + if( rc==SQLITE_BUSY ){ + /* Reset the return code so as not to report a checkpoint failure + ** just because there are active readers. */ + rc = SQLITE_OK; + } -#ifdef SQLITE_DEBUG -/* -**** This function may be used as part of assert() statements only. **** -** -** Return true if it would be illegal for pBtree to write into the -** table or index rooted at iRoot because other shared connections are -** simultaneously reading that same table or index. -** -** It is illegal for pBtree to write if some other Btree object that -** shares the same BtShared object is currently reading or writing -** the iRoot table. Except, if the other Btree object has the -** read-uncommitted flag set, then it is OK for the other object to -** have a read cursor. -** -** For example, before writing to any part of the table or index -** rooted at page iRoot, one should call: -** -** assert( !hasReadConflicts(pBtree, iRoot) ); -*/ -static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ - BtCursor *p; - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - if( p->pgnoRoot==iRoot - && p->pBtree!=pBtree - && 0==(p->pBtree->db->flags & SQLITE_ReadUncommitted) - ){ - return 1; + /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal + ** file has been copied into the database file, then block until all + ** readers have finished using the wal file. This ensures that the next + ** process to write to the database restarts the wal file. + */ + if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + assert( pWal->writeLock ); + if( pInfo->nBackfillhdr.mxFrame ){ + rc = SQLITE_BUSY; + }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + assert( mxSafeFrame==pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } } } - return 0; + + walcheckpoint_out: + walIteratorFree(pIter); + return rc; } -#endif /* #ifdef SQLITE_DEBUG */ /* -** Query to see if Btree handle p may obtain a lock of type eLock -** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return -** SQLITE_OK if the lock may be obtained (by calling -** setSharedCacheTableLock()), or SQLITE_LOCKED if not. +** Close a connection to a log file. */ -static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ - BtShared *pBt = p->pBt; - BtLock *pIter; - - assert( sqlite3BtreeHoldsMutex(p) ); - assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); - assert( p->db!=0 ); - assert( !(p->db->flags&SQLITE_ReadUncommitted)||eLock==WRITE_LOCK||iTab==1 ); - - /* If requesting a write-lock, then the Btree must have an open write - ** transaction on this file. And, obviously, for this to be so there - ** must be an open write transaction on the file itself. - */ - assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); - assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); - - /* This routine is a no-op if the shared-cache is not enabled */ - if( !p->sharable ){ - return SQLITE_OK; - } - - /* If some other connection is holding an exclusive lock, the - ** requested lock may not be obtained. - */ - if( pBt->pWriter!=p && pBt->isExclusive ){ - sqlite3ConnectionBlocked(p->db, pBt->pWriter->db); - return SQLITE_LOCKED_SHAREDCACHE; - } +SQLITE_PRIVATE int sqlite3WalClose( + Wal *pWal, /* Wal to close */ + int sync_flags, /* Flags to pass to OsSync() (or 0) */ + int nBuf, + u8 *zBuf /* Buffer of at least nBuf bytes */ +){ + int rc = SQLITE_OK; + if( pWal ){ + int isDelete = 0; /* True to unlink wal and wal-index files */ - for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - /* The condition (pIter->eLock!=eLock) in the following if(...) - ** statement is a simplification of: - ** - ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) + /* If an EXCLUSIVE lock can be obtained on the database file (using the + ** ordinary, rollback-mode locking methods, this guarantees that the + ** connection associated with this log file is the only connection to + ** the database. In this case checkpoint the database and unlink both + ** the wal and wal-index files. ** - ** since we know that if eLock==WRITE_LOCK, then no other connection - ** may hold a WRITE_LOCK on any table in this file (since there can - ** only be a single writer). + ** The EXCLUSIVE lock is not released before returning. */ - assert( pIter->eLock==READ_LOCK || pIter->eLock==WRITE_LOCK ); - assert( eLock==READ_LOCK || pIter->pBtree==p || pIter->eLock==READ_LOCK); - if( pIter->pBtree!=p && pIter->iTable==iTab && pIter->eLock!=eLock ){ - sqlite3ConnectionBlocked(p->db, pIter->pBtree->db); - if( eLock==WRITE_LOCK ){ - assert( p==pBt->pWriter ); - pBt->isPending = 1; + rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); + if( rc==SQLITE_OK ){ + if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - return SQLITE_LOCKED_SHAREDCACHE; + rc = sqlite3WalCheckpoint( + pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 + ); + if( rc==SQLITE_OK ){ + isDelete = 1; + } + } + + walIndexClose(pWal, isDelete); + sqlite3OsClose(pWal->pWalFd); + if( isDelete ){ + sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); } + WALTRACE(("WAL%p: closed\n", pWal)); + sqlite3_free((void *)pWal->apWiData); + sqlite3_free(pWal); } - return SQLITE_OK; + return rc; } -#endif /* !SQLITE_OMIT_SHARED_CACHE */ -#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Add a lock on the table with root-page iTable to the shared-btree used -** by Btree handle p. Parameter eLock must be either READ_LOCK or -** WRITE_LOCK. -** -** This function assumes the following: +** Try to read the wal-index header. Return 0 on success and 1 if +** there is a problem. ** -** (a) The specified Btree object p is connected to a sharable -** database (one with the BtShared.sharable flag set), and +** The wal-index is in shared memory. Another thread or process might +** be writing the header at the same time this procedure is trying to +** read it, which might result in inconsistency. A dirty read is detected +** by verifying that both copies of the header are the same and also by +** a checksum on the header. ** -** (b) No other Btree objects hold a lock that conflicts -** with the requested lock (i.e. querySharedCacheTableLock() has -** already been called and returned SQLITE_OK). +** If and only if the read is consistent and the header is different from +** pWal->hdr, then pWal->hdr is updated to the content of the new header +** and *pChanged is set to 1. ** -** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM -** is returned if a malloc attempt fails. +** If the checksum cannot be verified return non-zero. If the header +** is read successfully and the checksum verified, return zero. */ -static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ - BtShared *pBt = p->pBt; - BtLock *pLock = 0; - BtLock *pIter; - - assert( sqlite3BtreeHoldsMutex(p) ); - assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); - assert( p->db!=0 ); +static int walIndexTryHdr(Wal *pWal, int *pChanged){ + u32 aCksum[2]; /* Checksum on the header content */ + WalIndexHdr h1, h2; /* Two copies of the header content */ + WalIndexHdr volatile *aHdr; /* Header in shared memory */ - /* A connection with the read-uncommitted flag set will never try to - ** obtain a read-lock using this function. The only read-lock obtained - ** by a connection in read-uncommitted mode is on the sqlite_master - ** table, and that lock is obtained in BtreeBeginTrans(). */ - assert( 0==(p->db->flags&SQLITE_ReadUncommitted) || eLock==WRITE_LOCK ); + /* The first page of the wal-index must be mapped at this point. */ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); - /* This function should only be called on a sharable b-tree after it - ** has been determined that no other b-tree holds a conflicting lock. */ - assert( p->sharable ); - assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); + /* Read the header. This might happen concurrently with a write to the + ** same area of shared memory on a different CPU in a SMP, + ** meaning it is possible that an inconsistent snapshot is read + ** from the file. If this happens, return non-zero. + ** + ** There are two copies of the header at the beginning of the wal-index. + ** When reading, read [0] first then [1]. Writes are in the reverse order. + ** Memory barriers are used to prevent the compiler or the hardware from + ** reordering the reads and writes. + */ + aHdr = walIndexHdr(pWal); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + walShmBarrier(pWal); + memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); - /* First search the list for an existing lock on this table. */ - for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - if( pIter->iTable==iTable && pIter->pBtree==p ){ - pLock = pIter; - break; - } + if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ + return 1; /* Dirty read */ + } + if( h1.isInit==0 ){ + return 1; /* Malformed header - probably all zeros */ } - - /* If the above search did not find a BtLock struct associating Btree p - ** with table iTable, allocate one and link it into the list. - */ - if( !pLock ){ - pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); - if( !pLock ){ - return SQLITE_NOMEM; - } - pLock->iTable = iTable; - pLock->pBtree = p; - pLock->pNext = pBt->pLock; - pBt->pLock = pLock; + walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum); + if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){ + return 1; /* Checksum does not match */ } - /* Set the BtLock.eLock variable to the maximum of the current lock - ** and the requested lock. This means if a write-lock was already held - ** and a read-lock requested, we don't incorrectly downgrade the lock. - */ - assert( WRITE_LOCK>READ_LOCK ); - if( eLock>pLock->eLock ){ - pLock->eLock = eLock; + if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ + *pChanged = 1; + memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); + pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( pWal->szPage<=32768 ); + testcase( pWal->szPage>=65536 ); } - return SQLITE_OK; + /* The header was successfully read. Return zero. */ + return 0; } -#endif /* !SQLITE_OMIT_SHARED_CACHE */ -#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Release all the table locks (locks obtained via calls to -** the setSharedCacheTableLock() procedure) held by Btree object p. +** Read the wal-index header from the wal-index and into pWal->hdr. +** If the wal-header appears to be corrupt, try to reconstruct the +** wal-index from the WAL before returning. ** -** This function assumes that Btree p has an open read or write -** transaction. If it does not, then the BtShared.isPending variable -** may be incorrectly cleared. +** Set *pChanged to 1 if the wal-index header value in pWal->hdr is +** changed by this opertion. If pWal->hdr is unchanged, set *pChanged +** to 0. +** +** If the wal-index header is successfully read, return SQLITE_OK. +** Otherwise an SQLite error code. */ -static void clearAllSharedCacheTableLocks(Btree *p){ - BtShared *pBt = p->pBt; - BtLock **ppIter = &pBt->pLock; +static int walIndexReadHdr(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int badHdr; /* True if a header read failed */ + volatile u32 *page0; /* Chunk of wal-index containing header */ - assert( sqlite3BtreeHoldsMutex(p) ); - assert( p->sharable || 0==*ppIter ); - assert( p->inTrans>0 ); + /* Ensure that page 0 of the wal-index (the page that contains the + ** wal-index header) is mapped. Return early if an error occurs here. + */ + assert( pChanged ); + rc = walIndexPage(pWal, 0, &page0); + if( rc!=SQLITE_OK ){ + return rc; + }; + assert( page0 || pWal->writeLock==0 ); - while( *ppIter ){ - BtLock *pLock = *ppIter; - assert( pBt->isExclusive==0 || pBt->pWriter==pLock->pBtree ); - assert( pLock->pBtree->inTrans>=pLock->eLock ); - if( pLock->pBtree==p ){ - *ppIter = pLock->pNext; - assert( pLock->iTable!=1 || pLock==&p->lock ); - if( pLock->iTable!=1 ){ - sqlite3_free(pLock); + /* If the first page of the wal-index has been mapped, try to read the + ** wal-index header immediately, without holding any lock. This usually + ** works, but may fail if the wal-index header is corrupt or currently + ** being modified by another thread or process. + */ + badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); + + /* If the first attempt failed, it might have been due to a race + ** with a writer. So get a WRITE lock and try again. + */ + assert( badHdr==0 || pWal->writeLock==0 ); + if( badHdr ){ + if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; + } + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } } - }else{ - ppIter = &pLock->pNext; + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } } - assert( pBt->isPending==0 || pBt->pWriter ); - if( pBt->pWriter==p ){ - pBt->pWriter = 0; - pBt->isExclusive = 0; - pBt->isPending = 0; - }else if( pBt->nTransaction==2 ){ - /* This function is called when Btree p is concluding its - ** transaction. If there currently exists a writer, and p is not - ** that writer, then the number of locks held by connections other - ** than the writer must be about to drop to zero. In this case - ** set the isPending flag to 0. - ** - ** If there is not currently a writer, then BtShared.isPending must - ** be zero already. So this next line is harmless in that case. - */ - pBt->isPending = 0; - } -} - -/* -** This function changes all write-locks held by Btree p into read-locks. -*/ -static void downgradeAllSharedCacheTableLocks(Btree *p){ - BtShared *pBt = p->pBt; - if( pBt->pWriter==p ){ - BtLock *pLock; - pBt->pWriter = 0; - pBt->isExclusive = 0; - pBt->isPending = 0; - for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ - assert( pLock->eLock==READ_LOCK || pLock->pBtree==p ); - pLock->eLock = READ_LOCK; - } + /* If the header is read successfully, check the version number to make + ** sure the wal-index was not constructed with some future format that + ** this version of SQLite cannot understand. + */ + if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; } -} - -#endif /* SQLITE_OMIT_SHARED_CACHE */ - -static void releasePage(MemPage *pPage); /* Forward reference */ - -/* -***** This routine is used inside of assert() only **** -** -** Verify that the cursor holds the mutex on its BtShared -*/ -#ifdef SQLITE_DEBUG -static int cursorHoldsMutex(BtCursor *p){ - return sqlite3_mutex_held(p->pBt->mutex); -} -#endif - - -#ifndef SQLITE_OMIT_INCRBLOB -/* -** Invalidate the overflow page-list cache for cursor pCur, if any. -*/ -static void invalidateOverflowCache(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - sqlite3_free(pCur->aOverflow); - pCur->aOverflow = 0; -} -/* -** Invalidate the overflow page-list cache for all cursors opened -** on the shared btree structure pBt. -*/ -static void invalidateAllOverflowCache(BtShared *pBt){ - BtCursor *p; - assert( sqlite3_mutex_held(pBt->mutex) ); - for(p=pBt->pCursor; p; p=p->pNext){ - invalidateOverflowCache(p); - } + return rc; } /* -** This function is called before modifying the contents of a table -** to invalidate any incrblob cursors that are open on the -** row or one of the rows being modified. -** -** If argument isClearTable is true, then the entire contents of the -** table is about to be deleted. In this case invalidate all incrblob -** cursors open on any row within the table with root-page pgnoRoot. -** -** Otherwise, if argument isClearTable is false, then the row with -** rowid iRow is being replaced or deleted. In this case invalidate -** only those incrblob cursors open on that specific row. +** This is the value that walTryBeginRead returns when it needs to +** be retried. */ -static void invalidateIncrblobCursors( - Btree *pBtree, /* The database file to check */ - i64 iRow, /* The rowid that might be changing */ - int isClearTable /* True if all rows are being deleted */ -){ - BtCursor *p; - BtShared *pBt = pBtree->pBt; - assert( sqlite3BtreeHoldsMutex(pBtree) ); - for(p=pBt->pCursor; p; p=p->pNext){ - if( p->isIncrblobHandle && (isClearTable || p->info.nKey==iRow) ){ - p->eState = CURSOR_INVALID; - } - } -} - -#else - /* Stub functions when INCRBLOB is omitted */ - #define invalidateOverflowCache(x) - #define invalidateAllOverflowCache(x) - #define invalidateIncrblobCursors(x,y,z) -#endif /* SQLITE_OMIT_INCRBLOB */ +#define WAL_RETRY (-1) /* -** Set bit pgno of the BtShared.pHasContent bitvec. This is called -** when a page that previously contained data becomes a free-list leaf -** page. +** Attempt to start a read transaction. This might fail due to a race or +** other transient condition. When that happens, it returns WAL_RETRY to +** indicate to the caller that it is safe to retry immediately. ** -** The BtShared.pHasContent bitvec exists to work around an obscure -** bug caused by the interaction of two useful IO optimizations surrounding -** free-list leaf pages: +** On success return SQLITE_OK. On a permanent failure (such an +** I/O error or an SQLITE_BUSY because another process is running +** recovery) return a positive error code. ** -** 1) When all data is deleted from a page and the page becomes -** a free-list leaf page, the page is not written to the database -** (as free-list leaf pages contain no meaningful data). Sometimes -** such a page is not even journalled (as it will not be modified, -** why bother journalling it?). +** The useWal parameter is true to force the use of the WAL and disable +** the case where the WAL is bypassed because it has been completely +** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() +** to make a copy of the wal-index header into pWal->hdr. If the +** wal-index header has changed, *pChanged is set to 1 (as an indication +** to the caller that the local paget cache is obsolete and needs to be +** flushed.) When useWal==1, the wal-index header is assumed to already +** be loaded and the pChanged parameter is unused. ** -** 2) When a free-list leaf page is reused, its content is not read -** from the database or written to the journal file (why should it -** be, if it is not at all meaningful?). +** The caller must set the cnt parameter to the number of prior calls to +** this routine during the current read attempt that returned WAL_RETRY. +** This routine will start taking more aggressive measures to clear the +** race conditions after multiple WAL_RETRY returns, and after an excessive +** number of errors will ultimately return SQLITE_PROTOCOL. The +** SQLITE_PROTOCOL return indicates that some other process has gone rogue +** and is not honoring the locking protocol. There is a vanishingly small +** chance that SQLITE_PROTOCOL could be returned because of a run of really +** bad luck when there is lots of contention for the wal-index, but that +** possibility is so small that it can be safely neglected, we believe. ** -** By themselves, these optimizations work fine and provide a handy -** performance boost to bulk delete or insert operations. However, if -** a page is moved to the free-list and then reused within the same -** transaction, a problem comes up. If the page is not journalled when -** it is moved to the free-list and it is also not journalled when it -** is extracted from the free-list and reused, then the original data -** may be lost. In the event of a rollback, it may not be possible -** to restore the database to its original configuration. +** On success, this routine obtains a read lock on +** WAL_READ_LOCK(pWal->readLock). The pWal->readLock integer is +** in the range 0 <= pWal->readLock < WAL_NREADER. If pWal->readLock==(-1) +** that means the Wal does not hold any read lock. The reader must not +** access any database page that is modified by a WAL frame up to and +** including frame number aReadMark[pWal->readLock]. The reader will +** use WAL frames up to and including pWal->hdr.mxFrame if pWal->readLock>0 +** Or if pWal->readLock==0, then the reader will ignore the WAL +** completely and get all content directly from the database file. +** If the useWal parameter is 1 then the WAL will never be ignored and +** this routine will always set pWal->readLock>0 on success. +** When the read transaction is completed, the caller must release the +** lock on WAL_READ_LOCK(pWal->readLock) and set pWal->readLock to -1. ** -** The solution is the BtShared.pHasContent bitvec. Whenever a page is -** moved to become a free-list leaf page, the corresponding bit is -** set in the bitvec. Whenever a leaf page is extracted from the free-list, -** optimization 2 above is omitted if the corresponding bit is already -** set in BtShared.pHasContent. The contents of the bitvec are cleared -** at the end of every transaction. +** This routine uses the nBackfill and aReadMark[] fields of the header +** to select a particular WAL_READ_LOCK() that strives to let the +** checkpoint process do as much work as possible. This routine might +** update values of the aReadMark[] array in the header, but if it does +** so it takes care to hold an exclusive lock on the corresponding +** WAL_READ_LOCK() while changing values. */ -static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ - int rc = SQLITE_OK; - if( !pBt->pHasContent ){ - assert( pgno<=pBt->nPage ); - pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); - if( !pBt->pHasContent ){ - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ - rc = sqlite3BitvecSet(pBt->pHasContent, pgno); - } - return rc; -} +static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ + volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ + u32 mxReadMark; /* Largest aReadMark[] value */ + int mxI; /* Index of largest aReadMark[] value */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ -/* -** Query the BtShared.pHasContent vector. -** -** This function is called when a free-list leaf page is removed from the -** free-list for reuse. It returns false if it is safe to retrieve the -** page from the pager layer with the 'no-content' flag set. True otherwise. -*/ -static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ - Bitvec *p = pBt->pHasContent; - return (p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTest(p, pgno))); -} + assert( pWal->readLock<0 ); /* Not currently locked */ -/* -** Clear (destroy) the BtShared.pHasContent bitvec. This should be -** invoked at the conclusion of each write-transaction. -*/ -static void btreeClearHasContent(BtShared *pBt){ - sqlite3BitvecDestroy(pBt->pHasContent); - pBt->pHasContent = 0; -} + /* Take steps to avoid spinning forever if there is a protocol error. + ** + ** Circumstances that cause a RETRY should only last for the briefest + ** instances of time. No I/O or other system calls are done while the + ** locks are held, so the locks should not be held for very long. But + ** if we are unlucky, another process that is holding a lock might get + ** paged out or take a page-fault that is time-consuming to resolve, + ** during the few nanoseconds that it is holding the lock. In that case, + ** it might take longer than normal for the lock to free. + ** + ** After 5 RETRYs, we begin calling sqlite3OsSleep(). The first few + ** calls to sqlite3OsSleep() have a delay of 1 microsecond. Really this + ** is more of a scheduler yield than an actual delay. But on the 10th + ** an subsequent retries, the delays start becoming longer and longer, + ** so that on the 100th (and last) RETRY we delay for 21 milliseconds. + ** The total delay time before giving up is less than 1 second. + */ + if( cnt>5 ){ + int nDelay = 1; /* Pause time in microseconds */ + if( cnt>100 ){ + VVA_ONLY( pWal->lockError = 1; ) + return SQLITE_PROTOCOL; + } + if( cnt>=10 ) nDelay = (cnt-9)*238; /* Max delay 21ms. Total delay 996ms */ + sqlite3OsSleep(pWal->pVfs, nDelay); + } -/* -** Save the current cursor position in the variables BtCursor.nKey -** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. -** -** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) -** prior to calling this routine. -*/ -static int saveCursorPosition(BtCursor *pCur){ - int rc; - - assert( CURSOR_VALID==pCur->eState ); - assert( 0==pCur->pKey ); - assert( cursorHoldsMutex(pCur) ); - - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ + if( !useWal ){ + rc = walIndexReadHdr(pWal, pChanged); + if( rc==SQLITE_BUSY ){ + /* If there is not a recovery running in another thread or process + ** then convert BUSY errors to WAL_RETRY. If recovery is known to + ** be running, convert BUSY to BUSY_RECOVERY. There is a race here + ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY + ** would be technically correct. But the race is benign since with + ** WAL_RETRY this routine will be called again and will probably be + ** right on the second iteration. + */ + if( pWal->apWiData[0]==0 ){ + /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. + ** We assume this is a transient condition, so return WAL_RETRY. The + ** xShmMap() implementation used by the default unix and win32 VFS + ** modules may return SQLITE_BUSY due to a race condition in the + ** code that determines whether or not the shared-memory region + ** must be zeroed before the requested page is returned. + */ + rc = WAL_RETRY; + }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ + walUnlockShared(pWal, WAL_RECOVER_LOCK); + rc = WAL_RETRY; + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_BUSY_RECOVERY; + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + } - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ - if( 0==pCur->apPage[0]->intKey ){ - void *pKey = sqlite3Malloc( (int)pCur->nKey ); - if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); - if( rc==SQLITE_OK ){ - pCur->pKey = pKey; - }else{ - sqlite3_free(pKey); + pInfo = walCkptInfo(pWal); + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ + /* The WAL has been completely backfilled (or it is empty). + ** and can be safely ignored. + */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + walShmBarrier(pWal); + if( rc==SQLITE_OK ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + /* It is not safe to allow the reader to continue here if frames + ** may have been appended to the log before READ_LOCK(0) was obtained. + ** When holding READ_LOCK(0), the reader ignores the entire log file, + ** which implies that the database file contains a trustworthy + ** snapshoT. Since holding READ_LOCK(0) prevents a checkpoint from + ** happening, this is usually correct. + ** + ** However, if frames have been appended to the log (or if the log + ** is wrapped and written for that matter) before the READ_LOCK(0) + ** is obtained, that is not necessarily true. A checkpointer may + ** have started to backfill the appended frames but crashed before + ** it finished. Leaving a corrupt image in the database file. + */ + walUnlockShared(pWal, WAL_READ_LOCK(0)); + return WAL_RETRY; } - }else{ - rc = SQLITE_NOMEM; + pWal->readLock = 0; + return SQLITE_OK; + }else if( rc!=SQLITE_BUSY ){ + return rc; } } - assert( !pCur->apPage[0]->intKey || !pCur->pKey ); - if( rc==SQLITE_OK ){ - int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; + /* If we get this far, it means that the reader will want to use + ** the WAL to get at content from recent commits. The job now is + ** to select one of the aReadMark[] entries that is closest to + ** but not exceeding pWal->hdr.mxFrame and lock that entry. + */ + mxReadMark = 0; + mxI = 0; + for(i=1; iaReadMark[i]; + if( mxReadMark<=thisMark && thisMark<=pWal->hdr.mxFrame ){ + assert( thisMark!=READMARK_NOT_USED ); + mxReadMark = thisMark; + mxI = i; } - pCur->iPage = -1; - pCur->eState = CURSOR_REQUIRESEEK; } + /* There was once an "if" here. The extra "{" is to preserve indentation. */ + { + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ + for(i=1; iaReadMark[i] = pWal->hdr.mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + } - invalidateOverflowCache(pCur); + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** This does not guarantee that the copy of the wal-index header is up to + ** date before proceeding. That would not be possible without somehow + ** blocking writers. It only guarantees that a dangerous checkpoint or + ** log-wrap (either of which would require an exclusive lock on + ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. + */ + walShmBarrier(pWal); + if( pInfo->aReadMark[mxI]!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; + } + } return rc; } /* -** Save the positions of all cursors (except pExcept) that are open on -** the table with root-page iRoot. Usually, this is called just before cursor -** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()). +** Begin a read transaction on the database. +** +** This routine used to be called sqlite3OpenSnapshot() and with good reason: +** it takes a snapshot of the state of the WAL and wal-index for the current +** instant in time. The current thread will continue to use this snapshot. +** Other threads might append new content to the WAL and wal-index but +** that extra content is ignored by the current thread. +** +** If the database contents have changes since the previous read +** transaction, then *pChanged is set to 1 before returning. The +** Pager layer will use this to know that is cache is stale and +** needs to be flushed. */ -static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ - BtCursor *p; - assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pExcept==0 || pExcept->pBt==pBt ); - for(p=pBt->pCursor; p; p=p->pNext){ - if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && - p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); - if( SQLITE_OK!=rc ){ - return rc; - } - } - } - return SQLITE_OK; +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int cnt = 0; /* Number of TryBeginRead attempts */ + + do{ + rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); + }while( rc==WAL_RETRY ); + testcase( (rc&0xff)==SQLITE_BUSY ); + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; } /* -** Clear the current cursor position. +** Finish with a read transaction. All this does is release the +** read-lock. */ -SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - sqlite3_free(pCur->pKey); - pCur->pKey = 0; - pCur->eState = CURSOR_INVALID; +SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); + if( pWal->readLock>=0 ){ + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->readLock = -1; + } } /* -** In this version of BtreeMoveto, pKey is a packed index record -** such as is generated by the OP_MakeRecord opcode. Unpack the -** record and then call BtreeMovetoUnpacked() to do the work. +** Read a page from the WAL, if it is present in the WAL and if the +** current read transaction is configured to use the WAL. +** +** The *pInWal is set to 1 if the requested page is in the WAL and +** has been loaded. Or *pInWal is set to 0 if the page was not in +** the WAL and needs to be read out of the database. */ -static int btreeMoveto( - BtCursor *pCur, /* Cursor open on the btree to be searched */ - const void *pKey, /* Packed key if the btree is an index */ - i64 nKey, /* Integer key for tables. Size of pKey for indices */ - int bias, /* Bias search to the high end */ - int *pRes /* Write search results here */ +SQLITE_PRIVATE int sqlite3WalRead( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + int *pInWal, /* OUT: True if data is read from WAL */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ ){ - int rc; /* Status code */ - UnpackedRecord *pIdxKey; /* Unpacked index key */ - char aSpace[150]; /* Temp space for pIdxKey - to avoid a malloc */ + u32 iRead = 0; /* If !=0, WAL frame to return data from */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ + int iHash; /* Used to loop through N hash tables */ - if( pKey ){ - assert( nKey==(i64)(int)nKey ); - pIdxKey = sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, - aSpace, sizeof(aSpace)); - if( pIdxKey==0 ) return SQLITE_NOMEM; - }else{ - pIdxKey = 0; + /* This routine is only be called from within a read transaction. */ + assert( pWal->readLock>=0 || pWal->lockError ); + + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. + */ + if( iLast==0 || pWal->readLock==0 ){ + *pInWal = 0; + return SQLITE_OK; } - rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); - if( pKey ){ - sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + + /* Search the hash table or tables for an entry matching page number + ** pgno. Each iteration of the following for() loop searches one + ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). + ** + ** This code might run concurrently to the code in walIndexAppend() + ** that adds entries to the wal-index (and possibly to this hash + ** table). This means the value just read from the hash + ** slot (aHash[iKey]) may have been added before or after the + ** current read transaction was opened. Values added after the + ** read transaction was opened may have been written incorrectly - + ** i.e. these slots may contain garbage data. However, we assume + ** that any slots written before the current read transaction was + ** opened remain unmodified. + ** + ** For the reasons above, the if(...) condition featured in the inner + ** loop of the following block is more stringent that would be required + ** if we had exclusive access to the hash-table: + ** + ** (aPgno[iFrame]==pgno): + ** This condition filters out normal hash-table collisions. + ** + ** (iFrame<=iLast): + ** This condition filters out entries that were added to the hash + ** table after the current read-transaction had started. + */ + for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + volatile ht_slot *aHash; /* Pointer to hash table */ + volatile u32 *aPgno; /* Pointer to array of page numbers */ + u32 iZero; /* Frame number corresponding to aPgno[0] */ + int iKey; /* Hash slot index */ + int nCollide; /* Number of hash collisions remaining */ + int rc; /* Error code */ + + rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + if( rc!=SQLITE_OK ){ + return rc; + } + nCollide = HASHTABLE_NSLOT; + for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ + u32 iFrame = aHash[iKey] + iZero; + if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ + assert( iFrame>iRead ); + iRead = iFrame; + } + if( (nCollide--)==0 ){ + return SQLITE_CORRUPT_BKPT; + } + } } - return rc; + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* If expensive assert() statements are available, do a linear search + ** of the wal-index file content. Make sure the results agree with the + ** result obtained using the hash indexes above. */ + { + u32 iRead2 = 0; + u32 iTest; + for(iTest=iLast; iTest>0; iTest--){ + if( walFramePgno(pWal, iTest)==pgno ){ + iRead2 = iTest; + break; + } + } + assert( iRead==iRead2 ); + } +#endif + + /* If iRead is non-zero, then it is the log frame number that contains the + ** required page. Read and return data from the log file. + */ + if( iRead ){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + *pInWal = 1; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); + } + + *pInWal = 0; + return SQLITE_OK; } -/* -** Restore the cursor to the position it was in (or as close to as possible) -** when saveCursorPosition() was called. Note that this call deletes the -** saved position info stored by saveCursorPosition(), so there can be -** at most one effective restoreCursorPosition() call after each -** saveCursorPosition(). + +/* +** Return the size of the database in pages (or zero, if unknown). */ -static int btreeRestoreCursorPosition(BtCursor *pCur){ - int rc; - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState>=CURSOR_REQUIRESEEK ); - if( pCur->eState==CURSOR_FAULT ){ - return pCur->skipNext; - } - pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext); - if( rc==SQLITE_OK ){ - sqlite3_free(pCur->pKey); - pCur->pKey = 0; - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && ALWAYS(pWal->readLock>=0) ){ + return pWal->hdr.nPage; } - return rc; + return 0; } -#define restoreCursorPosition(p) \ - (p->eState>=CURSOR_REQUIRESEEK ? \ - btreeRestoreCursorPosition(p) : \ - SQLITE_OK) -/* -** Determine whether or not a cursor has moved from the position it -** was last placed at. Cursors can move when the row they are pointing -** at is deleted out from under them. +/* +** This function starts a write transaction on the WAL. ** -** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set to one if the cursor has moved and 0 if not. +** A read transaction must have already been started by a prior call +** to sqlite3WalBeginReadTransaction(). +** +** If another thread or process has written into the database since +** the read transaction was started, then it is not possible for this +** thread to write as doing so would cause a fork. So this routine +** returns SQLITE_BUSY in that case and no write transaction is started. +** +** There can only be a single writer active at a time. */ -SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ +SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; - rc = restoreCursorPosition(pCur); + /* Cannot start a write transaction without first holding a read + ** transaction. */ + assert( pWal->readLock>=0 ); + + if( pWal->readOnly ){ + return SQLITE_READONLY; + } + + /* Only one writer allowed at a time. Get the write lock. Return + ** SQLITE_BUSY if unable. + */ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); if( rc ){ - *pHasMoved = 1; return rc; } - if( pCur->eState!=CURSOR_VALID || pCur->skipNext!=0 ){ - *pHasMoved = 1; - }else{ - *pHasMoved = 0; + pWal->writeLock = 1; + + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. + */ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + rc = SQLITE_BUSY; } - return SQLITE_OK; + + return rc; } -#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Given a page number of a regular database page, return the page -** number for the pointer-map page that contains the entry for the -** input page number. -** -** Return 0 (not a valid page) for pgno==1 since there is -** no pointer map associated with page 1. The integrity_check logic -** requires that ptrmapPageno(*,1)!=1. +** End a write transaction. The commit has already been done. This +** routine merely releases the lock. */ -static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ - int nPagesPerMapPage; - Pgno iPtrMap, ret; - assert( sqlite3_mutex_held(pBt->mutex) ); - if( pgno<2 ) return 0; - nPagesPerMapPage = (pBt->usableSize/5)+1; - iPtrMap = (pgno-2)/nPagesPerMapPage; - ret = (iPtrMap*nPagesPerMapPage) + 2; - if( ret==PENDING_BYTE_PAGE(pBt) ){ - ret++; +SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ + if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; } - return ret; + return SQLITE_OK; } /* -** Write an entry into the pointer map. +** If any data has been written (but not committed) to the log file, this +** function moves the write-pointer back to the start of the transaction. ** -** This routine updates the pointer map entry for page number 'key' -** so that it maps to type 'eType' and parent page number 'pgno'. +** Additionally, the callback function is invoked for each frame written +** to the WAL since the start of the transaction. If the callback returns +** other than SQLITE_OK, it is not invoked again and the error code is +** returned to the caller. ** -** If *pRC is initially non-zero (non-SQLITE_OK) then this routine is -** a no-op. If an error occurs, the appropriate error code is written -** into *pRC. +** Otherwise, if the callback function does not return an error, this +** function returns SQLITE_OK. */ -static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ - DbPage *pDbPage; /* The pointer map page */ - u8 *pPtrmap; /* The pointer map data */ - Pgno iPtrmap; /* The pointer map page number */ - int offset; /* Offset in pointer map page */ - int rc; /* Return code from subfunctions */ - - if( *pRC ) return; - - assert( sqlite3_mutex_held(pBt->mutex) ); - /* The master-journal page number must never be used as a pointer map page */ - assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); +SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ + int rc = SQLITE_OK; + if( ALWAYS(pWal->writeLock) ){ + Pgno iMax = pWal->hdr.mxFrame; + Pgno iFrame; + + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. + */ + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); - assert( pBt->autoVacuum ); - if( key==0 ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; - } - iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); - if( rc!=SQLITE_OK ){ - *pRC = rc; - return; - } - offset = PTRMAP_PTROFFSET(iPtrmap, key); - if( offset<0 ){ - *pRC = SQLITE_CORRUPT_BKPT; - goto ptrmap_exit; - } - pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); - - if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); - *pRC= rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - pPtrmap[offset] = eType; - put4byte(&pPtrmap[offset+1], parent); - } + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + } + walCleanupHash(pWal); } + assert( rc==SQLITE_OK ); + return rc; +} -ptrmap_exit: - sqlite3PagerUnref(pDbPage); +/* +** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 +** values. This function populates the array with values required to +** "rollback" the write position of the WAL handle back to the current +** point in the event of a savepoint rollback (via WalSavepointUndo()). +*/ +SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ + assert( pWal->writeLock ); + aWalData[0] = pWal->hdr.mxFrame; + aWalData[1] = pWal->hdr.aFrameCksum[0]; + aWalData[2] = pWal->hdr.aFrameCksum[1]; + aWalData[3] = pWal->nCkpt; } -/* -** Read an entry from the pointer map. -** -** This routine retrieves the pointer map entry for page 'key', writing -** the type and parent page number to *pEType and *pPgno respectively. -** An error code is returned if something goes wrong, otherwise SQLITE_OK. +/* +** Move the write position of the WAL back to the point identified by +** the values in the aWalData[] array. aWalData must point to an array +** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated +** by a call to WalSavepoint(). */ -static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ - DbPage *pDbPage; /* The pointer map page */ - int iPtrmap; /* Pointer map page index */ - u8 *pPtrmap; /* Pointer map page data */ - int offset; /* Offset of entry in pointer map */ - int rc; +SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ + int rc = SQLITE_OK; - assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pWal->writeLock ); + assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); - iPtrmap = PTRMAP_PAGENO(pBt, key); - rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); - if( rc!=0 ){ - return rc; + if( aWalData[3]!=pWal->nCkpt ){ + /* This savepoint was opened immediately after the write-transaction + ** was started. Right after that, the writer decided to wrap around + ** to the start of the log. Update the savepoint values to match. + */ + aWalData[0] = 0; + aWalData[3] = pWal->nCkpt; } - pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); - offset = PTRMAP_PTROFFSET(iPtrmap, key); - assert( pEType!=0 ); - *pEType = pPtrmap[offset]; - if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); + if( aWalData[0]hdr.mxFrame ){ + pWal->hdr.mxFrame = aWalData[0]; + pWal->hdr.aFrameCksum[0] = aWalData[1]; + pWal->hdr.aFrameCksum[1] = aWalData[2]; + walCleanupHash(pWal); + } - sqlite3PagerUnref(pDbPage); - if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_BKPT; - return SQLITE_OK; + return rc; } -#else /* if defined SQLITE_OMIT_AUTOVACUUM */ - #define ptrmapPut(w,x,y,z,rc) - #define ptrmapGet(w,x,y,z) SQLITE_OK - #define ptrmapPutOvflPtr(x, y, rc) -#endif - /* -** Given a btree page and a cell index (0 means the first cell on -** the page, 1 means the second cell, and so forth) return a pointer -** to the cell content. +** This function is called just before writing a set of frames to the log +** file (see sqlite3WalFrames()). It checks to see if, instead of appending +** to the current log file, it is possible to overwrite the start of the +** existing log file with the new frames (i.e. "reset" the log). If so, +** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left +** unchanged. ** -** This routine works only for pages that do not contain overflow cells. +** SQLITE_OK is returned if no error is encountered (regardless of whether +** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned +** if an error occurs. */ -#define findCell(P,I) \ - ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) +static int walRestartLog(Wal *pWal){ + int rc = SQLITE_OK; + int cnt; -/* -** This a more complex version of findCell() that works for -** pages that do contain overflow cells. -*/ -static u8 *findOverflowCell(MemPage *pPage, int iCell){ - int i; - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - for(i=pPage->nOverflow-1; i>=0; i--){ - int k; - struct _OvflCell *pOvfl; - pOvfl = &pPage->aOvfl[i]; - k = pOvfl->idx; - if( k<=iCell ){ - if( k==iCell ){ - return pOvfl->pCell; + if( pWal->readLock==0 ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); + if( pInfo->nBackfill>0 ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + /* If all readers are using WAL_READ_LOCK(0) (in other words if no + ** readers are currently using the WAL), then the transactions + ** frames will overwrite the start of the existing log. Update the + ** wal-index header to reflect this. + ** + ** In theory it would be Ok to update the cache of the header only + ** at this point. But updating the actual wal-index header is also + ** safe and means there is no special case for sqlite3WalUndo() + ** to handle if this transaction is rolled back. + */ + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + + /* Limit the size of WAL file if the journal_size_limit PRAGMA is + ** set to a non-negative value. Log errors encountered + ** during the truncation attempt. */ + if( pWal->mxWalSize>=0 ){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->pWalFd, &sz); + if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){ + rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } + } + + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + aSalt[1] = salt1; + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + for(i=1; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + }else if( rc!=SQLITE_BUSY ){ + return rc; } - iCell--; } + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); } - return findCell(pPage, iCell); + return rc; } -/* -** Parse a cell content block and fill in the CellInfo structure. There -** are two versions of this function. btreeParseCell() takes a -** cell index as the second argument and btreeParseCellPtr() -** takes a pointer to the body of the cell as its second argument. -** -** Within this file, the parseCell() macro can be called instead of -** btreeParseCellPtr(). Using some compilers, this will be faster. +/* +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalBeginWriteTransaction()). */ -static void btreeParseCellPtr( - MemPage *pPage, /* Page containing the cell */ - u8 *pCell, /* Pointer to the cell text. */ - CellInfo *pInfo /* Fill in this structure */ +SQLITE_PRIVATE int sqlite3WalFrames( + Wal *pWal, /* Wal handle to write to */ + int szPage, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ ){ - u16 n; /* Number bytes in cell content header */ - u32 nPayload; /* Number of bytes of cell payload */ + int rc; /* Used to catch return codes */ + u32 iFrame; /* Next frame address */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ + PgHdr *p; /* Iterator to run through pList with. */ + PgHdr *pLast = 0; /* Last frame in list */ + int nLast = 0; /* Number of extra copies of last page */ - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pList ); + assert( pWal->writeLock ); - pInfo->pCell = pCell; - assert( pPage->leaf==0 || pPage->leaf==1 ); - n = pPage->childPtrSize; - assert( n==4-4*pPage->leaf ); - if( pPage->intKey ){ - if( pPage->hasData ){ - n += getVarint32(&pCell[n], nPayload); - }else{ - nPayload = 0; +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) + { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} + WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", + pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill")); + } +#endif + + /* See if it is possible to write these frames into the start of the + ** log file, instead of appending to it at pWal->hdr.mxFrame. + */ + if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ + return rc; + } + + /* If this is the first frame written into the log, write the WAL + ** header to the start of the WAL file. See comments at the top of + ** this source file for a description of the WAL header format. + */ + iFrame = pWal->hdr.mxFrame; + if( iFrame==0 ){ + u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ + u32 aCksum[2]; /* Checksum for wal-header */ + + sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); + sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); + sqlite3Put4byte(&aWalHdr[8], szPage); + sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); + sqlite3_randomness(8, pWal->hdr.aSalt); + memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); + walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); + sqlite3Put4byte(&aWalHdr[24], aCksum[0]); + sqlite3Put4byte(&aWalHdr[28], aCksum[1]); + + pWal->szPage = szPage; + pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; + pWal->hdr.aFrameCksum[0] = aCksum[0]; + pWal->hdr.aFrameCksum[1] = aCksum[1]; + + rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); + WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); + if( rc!=SQLITE_OK ){ + return rc; } - n += getVarint(&pCell[n], (u64*)&pInfo->nKey); - pInfo->nData = nPayload; - }else{ - pInfo->nData = 0; - n += getVarint32(&pCell[n], nPayload); - pInfo->nKey = nPayload; } - pInfo->nPayload = nPayload; - pInfo->nHeader = n; - testcase( nPayload==pPage->maxLocal ); - testcase( nPayload==pPage->maxLocal+1 ); - if( likely(nPayload<=pPage->maxLocal) ){ - /* This is the (easy) common case where the entire payload fits - ** on the local page. No overflow is required. - */ - int nSize; /* Total size of cell content in bytes */ - nSize = nPayload + n; - pInfo->nLocal = (u16)nPayload; - pInfo->iOverflow = 0; - if( (nSize & ~3)==0 ){ - nSize = 4; /* Minimum cell size is 4 */ + assert( (int)pWal->szPage==szPage ); + + /* Write the log file. */ + for(p=pList; p; p=p->pDirty){ + u32 nDbsize; /* Db-size field for frame header */ + i64 iOffset; /* Write offset in log file */ + void *pData; + + iOffset = walFrameOffset(++iFrame, szPage); + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + + /* Populate and write the frame header */ + nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0; +#if defined(SQLITE_HAS_CODEC) + if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; +#else + pData = p->pData; +#endif + walEncodeFrame(pWal, p->pgno, nDbsize, pData, aFrame); + rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset); + if( rc!=SQLITE_OK ){ + return rc; } - pInfo->nSize = (u16)nSize; - }else{ - /* If the payload will not fit completely on the local page, we have - ** to decide how much to store locally and how much to spill onto - ** overflow pages. The strategy is to minimize the amount of unused - ** space on overflow pages while keeping the amount of local storage - ** in between minLocal and maxLocal. - ** - ** Warning: changing the way overflow payload is distributed in any - ** way will result in an incompatible file format. - */ - int minLocal; /* Minimum amount of payload held locally */ - int maxLocal; /* Maximum amount of payload held locally */ - int surplus; /* Overflow payload available for local storage */ - minLocal = pPage->minLocal; - maxLocal = pPage->maxLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); - testcase( surplus==maxLocal ); - testcase( surplus==maxLocal+1 ); - if( surplus <= maxLocal ){ - pInfo->nLocal = (u16)surplus; - }else{ - pInfo->nLocal = (u16)minLocal; + /* Write the page data */ + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset+sizeof(aFrame)); + if( rc!=SQLITE_OK ){ + return rc; } - pInfo->iOverflow = (u16)(pInfo->nLocal + n); - pInfo->nSize = pInfo->iOverflow + 4; + pLast = p; } -} -#define parseCell(pPage, iCell, pInfo) \ - btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) -static void btreeParseCell( - MemPage *pPage, /* Page containing the cell */ - int iCell, /* The cell index. First cell is 0 */ - CellInfo *pInfo /* Fill in this structure */ -){ - parseCell(pPage, iCell, pInfo); -} -/* -** Compute the total number of bytes that a Cell needs in the cell -** data area of the btree-page. The return number includes the cell -** data header and the local payload, but not any overflow page or -** the space used by the cell pointer. -*/ -static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = &pCell[pPage->childPtrSize]; - u32 nSize; + /* Sync the log file if the 'isSync' flag was specified. */ + if( sync_flags ){ + i64 iSegment = sqlite3OsSectorSize(pWal->pWalFd); + i64 iOffset = walFrameOffset(iFrame+1, szPage); -#ifdef SQLITE_DEBUG - /* The value returned by this function should always be the same as - ** the (CellInfo.nSize) value found by doing a full parse of the - ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of - ** this function verifies that this invariant is not violated. */ - CellInfo debuginfo; - btreeParseCellPtr(pPage, pCell, &debuginfo); -#endif + assert( isCommit ); + assert( iSegment>0 ); - if( pPage->intKey ){ - u8 *pEnd; - if( pPage->hasData ){ - pIter += getVarint32(pIter, nSize); - }else{ - nSize = 0; + iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment); + while( iOffsetpData; +#endif + walEncodeFrame(pWal, pLast->pgno, nTruncate, pData, aFrame); + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset); + if( rc!=SQLITE_OK ){ + return rc; + } + iOffset += WAL_FRAME_HDRSIZE; + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset); + if( rc!=SQLITE_OK ){ + return rc; + } + nLast++; + iOffset += szPage; } - /* pIter now points at the 64-bit integer key value, a variable length - ** integer. The following block moves pIter to point at the first byte - ** past the end of the key value. */ - pEnd = &pIter[9]; - while( (*pIter++)&0x80 && pIterpWalFd, sync_flags); } - testcase( nSize==pPage->maxLocal ); - testcase( nSize==pPage->maxLocal+1 ); - if( nSize>pPage->maxLocal ){ - int minLocal = pPage->minLocal; - nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); - testcase( nSize==pPage->maxLocal ); - testcase( nSize==pPage->maxLocal+1 ); - if( nSize>pPage->maxLocal ){ - nSize = minLocal; - } - nSize += 4; + /* Append data to the wal-index. It is not necessary to lock the + ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index + ** guarantees that there are no other writers, and no data that may + ** be in use by existing readers is being overwritten. + */ + iFrame = pWal->hdr.mxFrame; + for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ + iFrame++; + rc = walIndexAppend(pWal, iFrame, p->pgno); } - nSize += (u32)(pIter - pCell); - - /* The minimum size of any cell is 4 bytes. */ - if( nSize<4 ){ - nSize = 4; + while( nLast>0 && rc==SQLITE_OK ){ + iFrame++; + nLast--; + rc = walIndexAppend(pWal, iFrame, pLast->pgno); } - assert( nSize==debuginfo.nSize ); - return (u16)nSize; -} + if( rc==SQLITE_OK ){ + /* Update the private copy of the header. */ + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + pWal->hdr.mxFrame = iFrame; + if( isCommit ){ + pWal->hdr.iChange++; + pWal->hdr.nPage = nTruncate; + } + /* If this is a commit, update the wal-index header too. */ + if( isCommit ){ + walIndexWriteHdr(pWal); + pWal->iCallback = iFrame; + } + } -#ifdef SQLITE_DEBUG -/* This variation on cellSizePtr() is used inside of assert() statements -** only. */ -static u16 cellSize(MemPage *pPage, int iCell){ - return cellSizePtr(pPage, findCell(pPage, iCell)); + WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); + return rc; } -#endif -#ifndef SQLITE_OMIT_AUTOVACUUM -/* -** If the cell pCell, part of page pPage contains a pointer -** to an overflow page, insert an entry into the pointer-map -** for the overflow page. +/* +** This routine is called to implement sqlite3_wal_checkpoint() and +** related interfaces. +** +** Obtain a CHECKPOINT lock and then backfill as much information as +** we can from WAL into the database. +** +** If parameter xBusy is not NULL, it is a pointer to a busy-handler +** callback. In this case this function runs a blocking checkpoint. */ -static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ - CellInfo info; - if( *pRC ) return; - assert( pCell!=0 ); - btreeParseCellPtr(pPage, pCell, &info); - assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); - if( info.iOverflow ){ - Pgno ovfl = get4byte(&pCell[info.iOverflow]); - ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); - } -} -#endif +SQLITE_PRIVATE int sqlite3WalCheckpoint( + Wal *pWal, /* Wal connection */ + int eMode, /* PASSIVE, FULL or RESTART */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of temporary buffer */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ +){ + int rc; /* Return code */ + int isChanged = 0; /* True if a new wal-index header is loaded */ + int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + assert( pWal->ckptLock==0 ); + assert( pWal->writeLock==0 ); -/* -** Defragment the page given. All Cells are moved to the -** end of the page and all free space is collected into one -** big FreeBlk that occurs in between the header and cell -** pointer array and the cell content area. -*/ -static int defragmentPage(MemPage *pPage){ - int i; /* Loop counter */ - int pc; /* Address of a i-th cell */ - int hdr; /* Offset to the page header */ - int size; /* Size of a cell */ - int usableSize; /* Number of usable bytes on a page */ - int cellOffset; /* Offset to the cell pointer array */ - int cbrk; /* Offset to the cell content area */ - int nCell; /* Number of cells on the page */ - unsigned char *data; /* The page data */ - unsigned char *temp; /* Temp area for cell content */ - int iCellFirst; /* First allowable cell index */ - int iCellLast; /* Last possible cell index */ + if( pWal->readOnly ) return SQLITE_READONLY; + WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + if( rc ){ + /* Usually this is SQLITE_BUSY meaning that another thread or process + ** is already running a checkpoint, or maybe a recovery. But it might + ** also be SQLITE_IOERR. */ + return rc; + } + pWal->ckptLock = 1; + /* If this is a blocking-checkpoint, then obtain the write-lock as well + ** to prevent any writers from running while the checkpoint is underway. + ** This has to be done before the call to walIndexReadHdr() below. + ** + ** If the writer lock cannot be obtained, then a passive checkpoint is + ** run instead. Since the checkpointer is not holding the writer lock, + ** there is no point in blocking waiting for any readers. Assuming no + ** other error occurs, this function will return SQLITE_BUSY to the caller. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + rc = SQLITE_OK; + } + } - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( pPage->pBt!=0 ); - assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; - hdr = pPage->hdrOffset; - cellOffset = pPage->cellOffset; - nCell = pPage->nCell; - assert( nCell==get2byte(&data[hdr+3]) ); - usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); - cbrk = usableSize; - iCellFirst = cellOffset + 2*nCell; - iCellLast = usableSize - 4; - for(i=0; iiCellLast ){ - return SQLITE_CORRUPT_BKPT; - } -#endif - assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); - cbrk -= size; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - if( cbrkusableSize ){ - return SQLITE_CORRUPT_BKPT; - } -#endif - assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); - testcase( cbrk+size==usableSize ); - testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); - put2byte(pAddr, cbrk); - } - assert( cbrk>=iCellFirst ); - put2byte(&data[hdr+5], cbrk); - data[hdr+1] = 0; - data[hdr+2] = 0; - data[hdr+7] = 0; - memset(&data[iCellFirst], 0, cbrk-iCellFirst); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; + /* Read the wal-index header. */ + if( rc==SQLITE_OK ){ + rc = walIndexReadHdr(pWal, &isChanged); } - return SQLITE_OK; -} - -/* -** Allocate nByte bytes of space from within the B-Tree page passed -** as the first argument. Write into *pIdx the index into pPage->aData[] -** of the first byte of allocated space. Return either SQLITE_OK or -** an error code (usually SQLITE_CORRUPT). -** -** The caller guarantees that there is sufficient space to make the -** allocation. This routine might need to defragment in order to bring -** all the space together, however. This routine will avoid using -** the first two bytes past the cell pointer area since presumably this -** allocation is being made in order to insert a new cell, so we will -** also end up needing a new cell pointer. -*/ -static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ - const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ - u8 * const data = pPage->aData; /* Local cache of pPage->aData */ - int nFrag; /* Number of fragmented bytes on pPage */ - int top; /* First byte of cell content area */ - int gap; /* First byte of gap between cell pointers and cell content */ - int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ - - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( pPage->pBt ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nByte>=0 ); /* Minimum cell size is 4 */ - assert( pPage->nFree>=nByte ); - assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); - nFrag = data[hdr+7]; - assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); - gap = pPage->cellOffset + 2*pPage->nCell; - top = get2byteNotZero(&data[hdr+5]); - if( gap>top ) return SQLITE_CORRUPT_BKPT; - testcase( gap+2==top ); - testcase( gap+1==top ); - testcase( gap==top ); + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); + } - if( nFrag>=60 ){ - /* Always defragment highly fragmented pages */ - rc = defragmentPage(pPage); - if( rc ) return rc; - top = get2byteNotZero(&data[hdr+5]); - }else if( gap+2<=top ){ - /* Search the freelist looking for a free slot big enough to satisfy - ** the request. The allocation is made from the first free slot in - ** the list that is large enough to accomadate it. - */ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] = (u8)(nFrag + x); - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + /* If no error occurred, set the output variables. */ + if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); } } - /* Check to make sure there is enough space in the gap to satisfy - ** the allocation. If not, defragment. - */ - testcase( gap+2+nByte==top ); - if( gap+2+nByte>top ){ - rc = defragmentPage(pPage); - if( rc ) return rc; - top = get2byteNotZero(&data[hdr+5]); - assert( gap+nByte<=top ); + if( isChanged ){ + /* If a new wal-index header was loaded before the checkpoint was + ** performed, then the pager-cache associated with pWal is now + ** out of date. So zero the cached wal-index header to ensure that + ** next time the pager opens a snapshot on this database it knows that + ** the cache needs to be reset. + */ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); } - - /* Allocate memory from the gap in between the cell pointer array - ** and the cell content area. The btreeInitPage() call has already - ** validated the freelist. Given that the freelist is valid, there - ** is no way that the allocation can extend off the end of the page. - ** The assert() below verifies the previous sentence. - */ - top -= nByte; - put2byte(&data[hdr+5], top); - assert( top+nByte <= pPage->pBt->usableSize ); - *pIdx = top; - return SQLITE_OK; + /* Release the locks. */ + sqlite3WalEndWriteTransaction(pWal); + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); + return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } -/* -** Return a section of the pPage->aData to the freelist. -** The first byte of the new free block is pPage->aDisk[start] -** and the size of the block is "size" bytes. -** -** Most of the effort here is involved in coalesing adjacent -** free blocks into a single big free block. +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. */ -static int freeSpace(MemPage *pPage, int start, int size){ - int addr, pbegin, hdr; - int iLast; /* Largest possible freeblock offset */ - unsigned char *data = pPage->aData; - - assert( pPage->pBt!=0 ); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( (start + size)<=pPage->pBt->usableSize ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( size>=0 ); /* Minimum cell size is 4 */ - - if( pPage->pBt->secureDelete ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[start], 0, size); - } - - /* Add the space back into the linked list of freeblocks. Note that - ** even though the freeblock list was checked by btreeInitPage(), - ** btreeInitPage() did not detect overlapping cells or - ** freeblocks that overlapped cells. Nor does it detect when the - ** cell content area exceeds the value in the page header. If these - ** situations arise, then subsequent insert operations might corrupt - ** the freelist. So we do need to check for corruption while scanning - ** the freelist. - */ - hdr = pPage->hdrOffset; - addr = hdr + 1; - iLast = pPage->pBt->usableSize - 4; - assert( start<=iLast ); - while( (pbegin = get2byte(&data[addr]))0 ){ - if( pbeginiLast ){ - return SQLITE_CORRUPT_BKPT; - } - assert( pbegin>addr || pbegin==0 ); - put2byte(&data[addr], start); - put2byte(&data[start], pbegin); - put2byte(&data[start+2], size); - pPage->nFree = pPage->nFree + (u16)size; - - /* Coalesce adjacent free blocks */ - addr = hdr + 1; - while( (pbegin = get2byte(&data[addr]))>0 ){ - int pnext, psize, x; - assert( pbegin>addr ); - assert( pbegin<=pPage->pBt->usableSize-4 ); - pnext = get2byte(&data[pbegin]); - psize = get2byte(&data[pbegin+2]); - if( pbegin + psize + 3 >= pnext && pnext>0 ){ - int frag = pnext - (pbegin+psize); - if( (frag<0) || (frag>(int)data[hdr+7]) ){ - return SQLITE_CORRUPT_BKPT; - } - data[hdr+7] -= (u8)frag; - x = get2byte(&data[pnext]); - put2byte(&data[pbegin], x); - x = pnext + get2byte(&data[pnext+2]) - pbegin; - put2byte(&data[pbegin+2], x); - }else{ - addr = pbegin; - } - } - - /* If the cell content area begins with a freeblock, remove it. */ - if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ - int top; - pbegin = get2byte(&data[hdr+1]); - memcpy(&data[hdr+1], &data[pbegin], 2); - top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); - put2byte(&data[hdr+5], top); +SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){ + u32 ret = 0; + if( pWal ){ + ret = pWal->iCallback; + pWal->iCallback = 0; } - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - return SQLITE_OK; + return (int)ret; } /* -** Decode the flags byte (the first byte of the header) for a page -** and initialize fields of the MemPage structure accordingly. +** This function is called to change the WAL subsystem into or out +** of locking_mode=EXCLUSIVE. ** -** Only the following combinations are supported. Anything different -** indicates a corrupt database files: +** If op is zero, then attempt to change from locking_mode=EXCLUSIVE +** into locking_mode=NORMAL. This means that we must acquire a lock +** on the pWal->readLock byte. If the WAL is already in locking_mode=NORMAL +** or if the acquisition of the lock fails, then return 0. If the +** transition out of exclusive-mode is successful, return 1. This +** operation must occur while the pager is still holding the exclusive +** lock on the main database file. ** -** PTF_ZERODATA -** PTF_ZERODATA | PTF_LEAF -** PTF_LEAFDATA | PTF_INTKEY -** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF -*/ -static int decodeFlags(MemPage *pPage, int flagByte){ - BtShared *pBt; /* A copy of pPage->pBt */ - - assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); - flagByte &= ~PTF_LEAF; - pPage->childPtrSize = 4-4*pPage->leaf; - pBt = pPage->pBt; - if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - pPage->intKey = 1; - pPage->hasData = pPage->leaf; - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - }else if( flagByte==PTF_ZERODATA ){ - pPage->intKey = 0; - pPage->hasData = 0; - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - }else{ - return SQLITE_CORRUPT_BKPT; - } - return SQLITE_OK; -} - -/* -** Initialize the auxiliary information for a disk block. +** If op is one, then change from locking_mode=NORMAL into +** locking_mode=EXCLUSIVE. This means that the pWal->readLock must +** be released. Return 1 if the transition is made and 0 if the +** WAL is already in exclusive-locking mode - meaning that this +** routine is a no-op. The pager must already hold the exclusive lock +** on the main database file before invoking this operation. ** -** Return SQLITE_OK on success. If we see that the page does -** not contain a well-formed database page, then return -** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not -** guarantee that the page is well-formed. It only shows that -** we failed to detect any corruption. +** If op is negative, then do a dry-run of the op==1 case but do +** not actually change anything. The pager uses this to see if it +** should acquire the database exclusive lock prior to invoking +** the op==1 case. */ -static int btreeInitPage(MemPage *pPage){ - - assert( pPage->pBt!=0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); - assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); - assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); - - if( !pPage->isInit ){ - u16 pc; /* Address of a freeblock within pPage->aData[] */ - u8 hdr; /* Offset to beginning of page header */ - u8 *data; /* Equal to pPage->aData */ - BtShared *pBt; /* The main btree structure */ - int usableSize; /* Amount of usable space on each page */ - u16 cellOffset; /* Offset from start of page to first cell pointer */ - int nFree; /* Number of unused bytes on the page */ - int top; /* First byte of the cell content area */ - int iCellFirst; /* First allowable cell or freeblock offset */ - int iCellLast; /* Last possible cell or freeblock offset */ - - pBt = pPage->pBt; - - hdr = pPage->hdrOffset; - data = pPage->aData; - if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; - assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); - pPage->maskPage = (u16)(pBt->pageSize - 1); - pPage->nOverflow = 0; - usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; - top = get2byteNotZero(&data[hdr+5]); - pPage->nCell = get2byte(&data[hdr+3]); - if( pPage->nCell>MX_CELL(pBt) ){ - /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_BKPT; - } - testcase( pPage->nCell==MX_CELL(pBt) ); - - /* A malformed database page might cause us to read past the end - ** of page when parsing a cell. - ** - ** The following block of code checks early to see if a cell extends - ** past the end of a page boundary and causes SQLITE_CORRUPT to be - ** returned if it does. - */ - iCellFirst = cellOffset + 2*pPage->nCell; - iCellLast = usableSize - 4; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - { - int i; /* Index into the cell pointer array */ - int sz; /* Size of a cell */ +SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ + int rc; + assert( pWal->writeLock==0 ); + assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); - if( !pPage->leaf ) iCellLast--; - for(i=0; inCell; i++){ - pc = get2byte(&data[cellOffset+i*2]); - testcase( pc==iCellFirst ); - testcase( pc==iCellLast ); - if( pciCellLast ){ - return SQLITE_CORRUPT_BKPT; - } - sz = cellSizePtr(pPage, &data[pc]); - testcase( pc+sz==usableSize ); - if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_BKPT; - } - } - if( !pPage->leaf ) iCellLast++; - } -#endif + /* pWal->readLock is usually set, but might be -1 if there was a + ** prior error while attempting to acquire are read-lock. This cannot + ** happen if the connection is actually in exclusive mode (as no xShmLock + ** locks are taken in this case). Nor should the pager attempt to + ** upgrade to exclusive-mode following such an error. + */ + assert( pWal->readLock>=0 || pWal->lockError ); + assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); - /* Compute the total free space on the page */ - pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; - while( pc>0 ){ - u16 next, size; - if( pciCellLast ){ - /* Start of free block is off the page */ - return SQLITE_CORRUPT_BKPT; - } - next = get2byte(&data[pc]); - size = get2byte(&data[pc+2]); - if( (next>0 && next<=pc+size+3) || pc+size>usableSize ){ - /* Free blocks must be in ascending order. And the last byte of - ** the free-block must lie on the database page. */ - return SQLITE_CORRUPT_BKPT; + if( op==0 ){ + if( pWal->exclusiveMode ){ + pWal->exclusiveMode = 0; + if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ + pWal->exclusiveMode = 1; } - nFree = nFree + size; - pc = next; - } - - /* At this point, nFree contains the sum of the offset to the start - ** of the cell-content area plus the number of free bytes within - ** the cell-content area. If this is greater than the usable-size - ** of the page, then the page must be corrupted. This check also - ** serves to verify that the offset to the start of the cell-content - ** area, according to the page header, lies within the page. - */ - if( nFree>usableSize ){ - return SQLITE_CORRUPT_BKPT; + rc = pWal->exclusiveMode==0; + }else{ + /* Already in locking_mode=NORMAL */ + rc = 0; } - pPage->nFree = (u16)(nFree - iCellFirst); - pPage->isInit = 1; + }else if( op>0 ){ + assert( pWal->exclusiveMode==0 ); + assert( pWal->readLock>=0 ); + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->exclusiveMode = 1; + rc = 1; + }else{ + rc = pWal->exclusiveMode==0; } - return SQLITE_OK; + return rc; } -/* -** Set up a raw page so that it looks like a database page holding -** no entries. +/* +** Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. */ -static void zeroPage(MemPage *pPage, int flags){ - unsigned char *data = pPage->aData; - BtShared *pBt = pPage->pBt; - u8 hdr = pPage->hdrOffset; - u16 first; - - assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); - assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage) == data ); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( sqlite3_mutex_held(pBt->mutex) ); - if( pBt->secureDelete ){ - memset(&data[hdr], 0, pBt->usableSize - hdr); - } - data[hdr] = (char)flags; - first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); - memset(&data[hdr+1], 0, 4); - data[hdr+7] = 0; - put2byte(&data[hdr+5], pBt->usableSize); - pPage->nFree = (u16)(pBt->usableSize - first); - decodeFlags(pPage, flags); - pPage->hdrOffset = hdr; - pPage->cellOffset = first; - pPage->nOverflow = 0; - assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); - pPage->maskPage = (u16)(pBt->pageSize - 1); - pPage->nCell = 0; - pPage->isInit = 1; +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ + return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +#endif /* #ifndef SQLITE_OMIT_WAL */ +/************** End of wal.c *************************************************/ +/************** Begin file btmutex.c *****************************************/ /* -** Convert a DbPage obtained from the pager into a MemPage used by -** the btree layer. -*/ -static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ - MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - pPage->aData = sqlite3PagerGetData(pDbPage); - pPage->pDbPage = pDbPage; - pPage->pBt = pBt; - pPage->pgno = pgno; - pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; - return pPage; -} - -/* -** Get a page from the pager. Initialize the MemPage.pBt and -** MemPage.aData elements if needed. +** 2007 August 27 ** -** If the noContent flag is set, it means that we do not care about -** the content of the page at this time. So do not go to the disk -** to fetch the content. Just fill in the content with zeros for now. -** If in the future we call sqlite3PagerWrite() on this page, that -** means we have started to be concerned about content and the disk -** read should occur at that point. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to implement mutexes on Btree objects. +** This code really belongs in btree.c. But btree.c is getting too +** big and we want to break it down some. This packaged seemed like +** a good breakout. +*/ +/************** Include btreeInt.h in the middle of btmutex.c ****************/ +/************** Begin file btreeInt.h ****************************************/ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a external (disk-based) database using BTrees. +** For a detailed discussion of BTrees, refer to +** +** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: +** "Sorting And Searching", pages 473-480. Addison-Wesley +** Publishing Company, Reading, Massachusetts. +** +** The basic idea is that each page of the file contains N database +** entries and N+1 pointers to subpages. +** +** ---------------------------------------------------------------- +** | Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) | +** ---------------------------------------------------------------- +** +** All of the keys on the page that Ptr(0) points to have values less +** than Key(0). All of the keys on page Ptr(1) and its subpages have +** values greater than Key(0) and less than Key(1). All of the keys +** on Ptr(N) and its subpages have values greater than Key(N-1). And +** so forth. +** +** Finding a particular key requires reading O(log(M)) pages from the +** disk where M is the number of entries in the tree. +** +** In this implementation, a single file can hold one or more separate +** BTrees. Each BTree is identified by the index of its root page. The +** key and data for any entry are combined to form the "payload". A +** fixed amount of payload can be carried directly on the database +** page. If the payload is larger than the preset amount then surplus +** bytes are stored on overflow pages. The payload for an entry +** and the preceding pointer are combined to form a "Cell". Each +** page has a small header which contains the Ptr(N) pointer and other +** information such as the size of key and data. +** +** FORMAT DETAILS +** +** The file is divided into pages. The first page is called page 1, +** the second is page 2, and so forth. A page number of zero indicates +** "no such page". The page size can be any power of 2 between 512 and 65536. +** Each page can be either a btree page, a freelist page, an overflow +** page, or a pointer-map page. +** +** The first page is always a btree page. The first 100 bytes of the first +** page contain a special header (the "file header") that describes the file. +** The format of the file header is as follows: +** +** OFFSET SIZE DESCRIPTION +** 0 16 Header string: "SQLite format 3\000" +** 16 2 Page size in bytes. +** 18 1 File format write version +** 19 1 File format read version +** 20 1 Bytes of unused space at the end of each page +** 21 1 Max embedded payload fraction +** 22 1 Min embedded payload fraction +** 23 1 Min leaf payload fraction +** 24 4 File change counter +** 28 4 Reserved for future use +** 32 4 First freelist page +** 36 4 Number of freelist pages in the file +** 40 60 15 4-byte meta values passed to higher layers +** +** 40 4 Schema cookie +** 44 4 File format of schema layer +** 48 4 Size of page cache +** 52 4 Largest root-page (auto/incr_vacuum) +** 56 4 1=UTF-8 2=UTF16le 3=UTF16be +** 60 4 User version +** 64 4 Incremental vacuum mode +** 68 4 unused +** 72 4 unused +** 76 4 unused +** +** All of the integer values are big-endian (most significant byte first). +** +** The file change counter is incremented when the database is changed +** This counter allows other processes to know when the file has changed +** and thus when they need to flush their cache. +** +** The max embedded payload fraction is the amount of the total usable +** space in a page that can be consumed by a single cell for standard +** B-tree (non-LEAFDATA) tables. A value of 255 means 100%. The default +** is to limit the maximum cell size so that at least 4 cells will fit +** on one page. Thus the default max embedded payload fraction is 64. +** +** If the payload for a cell is larger than the max payload, then extra +** payload is spilled to overflow pages. Once an overflow page is allocated, +** as many bytes as possible are moved into the overflow pages without letting +** the cell size drop below the min embedded payload fraction. +** +** The min leaf payload fraction is like the min embedded payload fraction +** except that it applies to leaf nodes in a LEAFDATA tree. The maximum +** payload fraction for a LEAFDATA tree is always 100% (or 255) and it +** not specified in the header. +** +** Each btree pages is divided into three sections: The header, the +** cell pointer array, and the cell content area. Page 1 also has a 100-byte +** file header that occurs before the page header. +** +** |----------------| +** | file header | 100 bytes. Page 1 only. +** |----------------| +** | page header | 8 bytes for leaves. 12 bytes for interior nodes +** |----------------| +** | cell pointer | | 2 bytes per cell. Sorted order. +** | array | | Grows downward +** | | v +** |----------------| +** | unallocated | +** | space | +** |----------------| ^ Grows upwards +** | cell content | | Arbitrary order interspersed with freeblocks. +** | area | | and free space fragments. +** |----------------| +** +** The page headers looks like this: +** +** OFFSET SIZE DESCRIPTION +** 0 1 Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf +** 1 2 byte offset to the first freeblock +** 3 2 number of cells on this page +** 5 2 first byte of the cell content area +** 7 1 number of fragmented free bytes +** 8 4 Right child (the Ptr(N) value). Omitted on leaves. +** +** The flags define the format of this btree page. The leaf flag means that +** this page has no children. The zerodata flag means that this page carries +** only keys and no data. The intkey flag means that the key is a integer +** which is stored in the key size entry of the cell header rather than in +** the payload area. +** +** The cell pointer array begins on the first byte after the page header. +** The cell pointer array contains zero or more 2-byte numbers which are +** offsets from the beginning of the page to the cell content in the cell +** content area. The cell pointers occur in sorted order. The system strives +** to keep free space after the last cell pointer so that new cells can +** be easily added without having to defragment the page. +** +** Cell content is stored at the very end of the page and grows toward the +** beginning of the page. +** +** Unused space within the cell content area is collected into a linked list of +** freeblocks. Each freeblock is at least 4 bytes in size. The byte offset +** to the first freeblock is given in the header. Freeblocks occur in +** increasing order. Because a freeblock must be at least 4 bytes in size, +** any group of 3 or fewer unused bytes in the cell content area cannot +** exist on the freeblock chain. A group of 3 or fewer free bytes is called +** a fragment. The total number of bytes in all fragments is recorded. +** in the page header at offset 7. +** +** SIZE DESCRIPTION +** 2 Byte offset of the next freeblock +** 2 Bytes in this freeblock +** +** Cells are of variable length. Cells are stored in the cell content area at +** the end of the page. Pointers to the cells are in the cell pointer array +** that immediately follows the page header. Cells is not necessarily +** contiguous or in order, but cell pointers are contiguous and in order. +** +** Cell content makes use of variable length integers. A variable +** length integer is 1 to 9 bytes where the lower 7 bits of each +** byte are used. The integer consists of all bytes that have bit 8 set and +** the first byte with bit 8 clear. The most significant byte of the integer +** appears first. A variable-length integer may not be more than 9 bytes long. +** As a special case, all 8 bytes of the 9th byte are used as data. This +** allows a 64-bit integer to be encoded in 9 bytes. +** +** 0x00 becomes 0x00000000 +** 0x7f becomes 0x0000007f +** 0x81 0x00 becomes 0x00000080 +** 0x82 0x00 becomes 0x00000100 +** 0x80 0x7f becomes 0x0000007f +** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 +** +** Variable length integers are used for rowids and to hold the number of +** bytes of key and data in a btree cell. +** +** The content of a cell looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of the left child. Omitted if leaf flag is set. +** var Number of bytes of data. Omitted if the zerodata flag is set. +** var Number of bytes of key. Or the key itself if intkey flag is set. +** * Payload +** 4 First page of the overflow chain. Omitted if no overflow +** +** Overflow pages form a linked list. Each page except the last is completely +** filled with data (pagesize - 4 bytes). The last page can have as little +** as 1 byte of data. +** +** SIZE DESCRIPTION +** 4 Page number of next overflow page +** * Data +** +** Freelist pages come in two subtypes: trunk pages and leaf pages. The +** file header points to the first in a linked list of trunk page. Each trunk +** page points to multiple leaf pages. The content of a leaf page is +** unspecified. A trunk page looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of next trunk page +** 4 Number of leaf pointers on this page +** * zero or more pages numbers of leaves */ -static int btreeGetPage( - BtShared *pBt, /* The btree */ - Pgno pgno, /* Number of the page to fetch */ - MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ -){ - int rc; - DbPage *pDbPage; - assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); - if( rc ) return rc; - *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); - return SQLITE_OK; -} -/* -** Retrieve a page from the pager cache. If the requested page is not -** already in the pager cache return NULL. Initialize the MemPage.pBt and -** MemPage.aData elements if needed. +/* The following value is the maximum cell size assuming a maximum page +** size give above. */ -static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ - DbPage *pDbPage; - assert( sqlite3_mutex_held(pBt->mutex) ); - pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); - if( pDbPage ){ - return btreePageFromDbPage(pDbPage, pgno, pBt); - } - return 0; -} +#define MX_CELL_SIZE(pBt) ((int)(pBt->pageSize-8)) -/* -** Return the size of the database file in pages. If there is any kind of -** error, return ((unsigned int)-1). +/* The maximum number of cells on a single page of the database. This +** assumes a minimum cell size of 6 bytes (4 bytes for the cell itself +** plus 2 bytes for the index to the cell in the page header). Such +** small cells will be rare, but they are possible. */ -static Pgno btreePagecount(BtShared *pBt){ - return pBt->nPage; -} -SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ - assert( sqlite3BtreeHoldsMutex(p) ); - assert( ((p->pBt->nPage)&0x8000000)==0 ); - return (int)btreePagecount(p->pBt); -} +#define MX_CELL(pBt) ((pBt->pageSize-8)/6) + +/* Forward declarations */ +typedef struct MemPage MemPage; +typedef struct BtLock BtLock; /* -** Get a page from the pager and initialize it. This routine is just a -** convenience wrapper around separate calls to btreeGetPage() and -** btreeInitPage(). +** This is a magic string that appears at the beginning of every +** SQLite database in order to identify the file as a real database. ** -** If an error occurs, then the value *ppPage is set to is undefined. It -** may remain unchanged, or it may be set to an invalid value. +** You can change this value at compile-time by specifying a +** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The +** header must be exactly 16 bytes including the zero-terminator so +** the string itself should be 15 characters long. If you change +** the header, then your custom library will not be able to read +** databases generated by the standard tools and the standard tools +** will not be able to read databases created by your custom library. */ -static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ -){ - int rc; - assert( sqlite3_mutex_held(pBt->mutex) ); - - if( pgno>btreePagecount(pBt) ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); - if( rc==SQLITE_OK ){ - rc = btreeInitPage(*ppPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - } - } - } - - testcase( pgno==0 ); - assert( pgno!=0 || rc==SQLITE_CORRUPT ); - return rc; -} +#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ +# define SQLITE_FILE_HEADER "SQLite format 3" +#endif /* -** Release a MemPage. This should be called once for each prior -** call to btreeGetPage. +** Page type flags. An ORed combination of these flags appear as the +** first byte of on-disk image of every BTree page. */ -static void releasePage(MemPage *pPage){ - if( pPage ){ - assert( pPage->aData ); - assert( pPage->pBt ); - assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - sqlite3PagerUnref(pPage->pDbPage); - } -} +#define PTF_INTKEY 0x01 +#define PTF_ZERODATA 0x02 +#define PTF_LEAFDATA 0x04 +#define PTF_LEAF 0x08 /* -** During a rollback, when the pager reloads information into the cache -** so that the cache is restored to its original state at the start of -** the transaction, for each page restored this routine is called. +** As each page of the file is loaded into memory, an instance of the following +** structure is appended and initialized to zero. This structure stores +** information about the page that is decoded from the raw file page. ** -** This routine needs to reset the extra data section at the end of the -** page to agree with the restored data. +** The pParent field points back to the parent page. This allows us to +** walk up the BTree from any leaf to the root. Care must be taken to +** unref() the parent page pointer when this page is no longer referenced. +** The pageDestructor() routine handles that chore. +** +** Access to all fields of this structure is controlled by the mutex +** stored in MemPage.pBt->mutex. */ -static void pageReinit(DbPage *pData){ - MemPage *pPage; - pPage = (MemPage *)sqlite3PagerGetExtra(pData); - assert( sqlite3PagerPageRefcount(pData)>0 ); - if( pPage->isInit ){ - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->isInit = 0; - if( sqlite3PagerPageRefcount(pData)>1 ){ - /* pPage might not be a btree page; it might be an overflow page - ** or ptrmap page or a free page. In those cases, the following - ** call to btreeInitPage() will likely return SQLITE_CORRUPT. - ** But no harm is done by this. And it is very important that - ** btreeInitPage() be called on every btree page so we make - ** the call for every page that comes in for re-initing. */ - btreeInitPage(pPage); - } - } -} +struct MemPage { + u8 isInit; /* True if previously initialized. MUST BE FIRST! */ + u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ + u8 intKey; /* True if intkey flag is set */ + u8 leaf; /* True if leaf flag is set */ + u8 hasData; /* True if this page stores data */ + u8 hdrOffset; /* 100 for page 1. 0 otherwise */ + u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ + u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ + u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ + u16 cellOffset; /* Index in aData of first cell pointer */ + u16 nFree; /* Number of free bytes on the page */ + u16 nCell; /* Number of cells on this page, local and ovfl */ + u16 maskPage; /* Mask for page offset */ + struct _OvflCell { /* Cells that will not fit on aData[] */ + u8 *pCell; /* Pointers to the body of the overflow cell */ + u16 idx; /* Insert this cell before idx-th non-overflow cell */ + } aOvfl[5]; + BtShared *pBt; /* Pointer to BtShared that this page is part of */ + u8 *aData; /* Pointer to disk image of the page data */ + DbPage *pDbPage; /* Pager page handle */ + Pgno pgno; /* Page number for this page */ +}; /* -** Invoke the busy handler for a btree. +** The in-memory image of a disk page has the auxiliary information appended +** to the end. EXTRA_SIZE is the number of bytes of space needed to hold +** that extra information. */ -static int btreeInvokeBusyHandler(void *pArg){ - BtShared *pBt = (BtShared*)pArg; - assert( pBt->db ); - assert( sqlite3_mutex_held(pBt->db->mutex) ); - return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); -} +#define EXTRA_SIZE sizeof(MemPage) /* -** Open a database file. -** -** zFilename is the name of the database file. If zFilename is NULL -** then an ephemeral database is created. The ephemeral database might -** be exclusively in memory, or it might use a disk-based memory cache. -** Either way, the ephemeral database will be automatically deleted -** when sqlite3BtreeClose() is called. +** A linked list of the following structures is stored at BtShared.pLock. +** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor +** is opened on the table with root page BtShared.iTable. Locks are removed +** from this list when a transaction is committed or rolled back, or when +** a btree handle is closed. +*/ +struct BtLock { + Btree *pBtree; /* Btree handle holding this lock */ + Pgno iTable; /* Root page of table */ + u8 eLock; /* READ_LOCK or WRITE_LOCK */ + BtLock *pNext; /* Next in BtShared.pLock list */ +}; + +/* Candidate values for BtLock.eLock */ +#define READ_LOCK 1 +#define WRITE_LOCK 2 + +/* A Btree handle ** -** If zFilename is ":memory:" then an in-memory database is created -** that is automatically destroyed when it is closed. +** A database connection contains a pointer to an instance of +** this object for every database file that it has open. This structure +** is opaque to the database connection. The database connection cannot +** see the internals of this structure and only deals with pointers to +** this structure. ** -** The "flags" parameter is a bitmask that might contain bits -** BTREE_OMIT_JOURNAL and/or BTREE_NO_READLOCK. The BTREE_NO_READLOCK -** bit is also set if the SQLITE_NoReadlock flags is set in db->flags. -** These flags are passed through into sqlite3PagerOpen() and must -** be the same values as PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK. +** For some database files, the same underlying database cache might be +** shared between multiple connections. In that case, each connection +** has it own instance of this object. But each instance of this object +** points to the same BtShared object. The database cache and the +** schema associated with the database file are all contained within +** the BtShared object. ** -** If the database is already opened in the same database connection -** and we are in shared cache mode, then the open will fail with an -** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared -** objects in the same database connection since doing so will lead -** to problems with locking. +** All fields in this structure are accessed under sqlite3.mutex. +** The pBt pointer itself may not be changed while there exists cursors +** in the referenced BtShared that point back to this Btree since those +** cursors have to go through this Btree to find their BtShared and +** they often do so without holding sqlite3.mutex. */ -SQLITE_PRIVATE int sqlite3BtreeOpen( - const char *zFilename, /* Name of the file containing the BTree database */ - sqlite3 *db, /* Associated database handle */ - Btree **ppBtree, /* Pointer to new Btree object written here */ - int flags, /* Options */ - int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ -){ - sqlite3_vfs *pVfs; /* The VFS to use for this btree */ - BtShared *pBt = 0; /* Shared part of btree structure */ - Btree *p; /* Handle to return */ - sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ - int rc = SQLITE_OK; /* Result code from this function */ - u8 nReserve; /* Byte of unused space on each page */ - unsigned char zDbHeader[100]; /* Database header content */ - - /* True if opening an ephemeral, temporary database */ - const int isTempDb = zFilename==0 || zFilename[0]==0; - - /* Set the variable isMemdb to true for an in-memory database, or - ** false for a file-based database. - */ -#ifdef SQLITE_OMIT_MEMORYDB - const int isMemdb = 0; -#else - const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) - || (isTempDb && sqlite3TempInMemory(db)); +struct Btree { + sqlite3 *db; /* The database connection holding this btree */ + BtShared *pBt; /* Sharable content of this btree */ + u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ + u8 sharable; /* True if we can share pBt with another db */ + u8 locked; /* True if db currently has pBt locked */ + int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ + int nBackup; /* Number of backup operations reading this btree */ + Btree *pNext; /* List of other sharable Btrees from the same db */ + Btree *pPrev; /* Back pointer of the same list */ +#ifndef SQLITE_OMIT_SHARED_CACHE + BtLock lock; /* Object used to lock page 1 */ #endif +}; - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); - assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ - - /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ - assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); - - /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ - assert( (flags & BTREE_SINGLE)==0 || isTempDb ); - - if( db->flags & SQLITE_NoReadlock ){ - flags |= BTREE_NO_READLOCK; - } - if( isMemdb ){ - flags |= BTREE_MEMORY; - } - if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ - vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; - } - pVfs = db->pVfs; - p = sqlite3MallocZero(sizeof(Btree)); - if( !p ){ - return SQLITE_NOMEM; - } - p->inTrans = TRANS_NONE; - p->db = db; -#ifndef SQLITE_OMIT_SHARED_CACHE - p->lock.pBtree = p; - p->lock.iTable = 1; -#endif +/* +** Btree.inTrans may take one of the following values. +** +** If the shared-data extension is enabled, there may be multiple users +** of the Btree structure. At most one of these may open a write transaction, +** but any number may have active read transactions. +*/ +#define TRANS_NONE 0 +#define TRANS_READ 1 +#define TRANS_WRITE 2 -#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) - /* - ** If this Btree is a candidate for shared cache, try to find an - ** existing BtShared object that we can share with - */ - if( isMemdb==0 && isTempDb==0 ){ - if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ - int nFullPathname = pVfs->mxPathname+1; - char *zFullPathname = sqlite3Malloc(nFullPathname); - sqlite3_mutex *mutexShared; - p->sharable = 1; - if( !zFullPathname ){ - sqlite3_free(p); - return SQLITE_NOMEM; - } - sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); - mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); - sqlite3_mutex_enter(mutexOpen); - mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); - sqlite3_mutex_enter(mutexShared); - for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ - assert( pBt->nRef>0 ); - if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager)) - && sqlite3PagerVfs(pBt->pPager)==pVfs ){ - int iDb; - for(iDb=db->nDb-1; iDb>=0; iDb--){ - Btree *pExisting = db->aDb[iDb].pBt; - if( pExisting && pExisting->pBt==pBt ){ - sqlite3_mutex_leave(mutexShared); - sqlite3_mutex_leave(mutexOpen); - sqlite3_free(zFullPathname); - sqlite3_free(p); - return SQLITE_CONSTRAINT; - } - } - p->pBt = pBt; - pBt->nRef++; - break; - } - } - sqlite3_mutex_leave(mutexShared); - sqlite3_free(zFullPathname); - } -#ifdef SQLITE_DEBUG - else{ - /* In debug mode, we mark all persistent databases as sharable - ** even when they are not. This exercises the locking code and - ** gives more opportunity for asserts(sqlite3_mutex_held()) - ** statements to find locking problems. - */ - p->sharable = 1; - } -#endif - } -#endif - if( pBt==0 ){ - /* - ** The following asserts make sure that structures used by the btree are - ** the right size. This is to guard against size changes that result - ** when compiling on a different architecture. - */ - assert( sizeof(i64)==8 || sizeof(i64)==4 ); - assert( sizeof(u64)==8 || sizeof(u64)==4 ); - assert( sizeof(u32)==4 ); - assert( sizeof(u16)==2 ); - assert( sizeof(Pgno)==4 ); - - pBt = sqlite3MallocZero( sizeof(*pBt) ); - if( pBt==0 ){ - rc = SQLITE_NOMEM; - goto btree_open_out; - } - rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, - EXTRA_SIZE, flags, vfsFlags, pageReinit); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); - } - if( rc!=SQLITE_OK ){ - goto btree_open_out; - } - pBt->openFlags = (u8)flags; - pBt->db = db; - sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); - p->pBt = pBt; - - pBt->pCursor = 0; - pBt->pPage1 = 0; - pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); -#ifdef SQLITE_SECURE_DELETE - pBt->secureDelete = 1; -#endif - pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); - if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE - || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ - pBt->pageSize = 0; -#ifndef SQLITE_OMIT_AUTOVACUUM - /* If the magic name ":memory:" will create an in-memory database, then - ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if - ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if - ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a - ** regular file-name. In this case the auto-vacuum applies as per normal. - */ - if( zFilename && !isMemdb ){ - pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); - pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); - } -#endif - nReserve = 0; - }else{ - nReserve = zDbHeader[20]; - pBt->pageSizeFixed = 1; +/* +** An instance of this object represents a single database file. +** +** A single database file can be in use as the same time by two +** or more database connections. When two or more connections are +** sharing the same database file, each connection has it own +** private Btree object for the file and each of those Btrees points +** to this one BtShared object. BtShared.nRef is the number of +** connections currently sharing this database file. +** +** Fields in this structure are accessed under the BtShared.mutex +** mutex, except for nRef and pNext which are accessed under the +** global SQLITE_MUTEX_STATIC_MASTER mutex. The pPager field +** may not be modified once it is initially set as long as nRef>0. +** The pSchema field may be set once under BtShared.mutex and +** thereafter is unchanged as long as nRef>0. +** +** isPending: +** +** If a BtShared client fails to obtain a write-lock on a database +** table (because there exists one or more read-locks on the table), +** the shared-cache enters 'pending-lock' state and isPending is +** set to true. +** +** The shared-cache leaves the 'pending lock' state when either of +** the following occur: +** +** 1) The current writer (BtShared.pWriter) concludes its transaction, OR +** 2) The number of locks held by other connections drops to zero. +** +** while in the 'pending-lock' state, no connection may start a new +** transaction. +** +** This feature is included to help prevent writer-starvation. +*/ +struct BtShared { + Pager *pPager; /* The page cache */ + sqlite3 *db; /* Database connection currently using this Btree */ + BtCursor *pCursor; /* A list of all open cursors */ + MemPage *pPage1; /* First page of the database */ + u8 readOnly; /* True if the underlying file is readonly */ + u8 pageSizeFixed; /* True if the page size can no longer be changed */ + u8 secureDelete; /* True if secure_delete is enabled */ + u8 initiallyEmpty; /* Database is empty at start of transaction */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ #ifndef SQLITE_OMIT_AUTOVACUUM - pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); - pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); -#endif - } - rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); - if( rc ) goto btree_open_out; - pBt->usableSize = pBt->pageSize - nReserve; - assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ - -#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) - /* Add the new BtShared object to the linked list sharable BtShareds. - */ - if( p->sharable ){ - sqlite3_mutex *mutexShared; - pBt->nRef = 1; - mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); - if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ - pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); - if( pBt->mutex==0 ){ - rc = SQLITE_NOMEM; - db->mallocFailed = 0; - goto btree_open_out; - } - } - sqlite3_mutex_enter(mutexShared); - pBt->pNext = GLOBAL(BtShared*,sqlite3SharedCacheList); - GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt; - sqlite3_mutex_leave(mutexShared); - } + u8 autoVacuum; /* True if auto-vacuum is enabled */ + u8 incrVacuum; /* True if incr-vacuum is enabled */ #endif - } - -#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) - /* If the new Btree uses a sharable pBtShared, then link the new - ** Btree into the list of all sharable Btrees for the same connection. - ** The list is kept in ascending order by pBt address. - */ - if( p->sharable ){ - int i; - Btree *pSib; - for(i=0; inDb; i++){ - if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ - while( pSib->pPrev ){ pSib = pSib->pPrev; } - if( p->pBtpBt ){ - p->pNext = pSib; - p->pPrev = 0; - pSib->pPrev = p; - }else{ - while( pSib->pNext && pSib->pNext->pBtpBt ){ - pSib = pSib->pNext; - } - p->pNext = pSib->pNext; - p->pPrev = pSib; - if( p->pNext ){ - p->pNext->pPrev = p; - } - pSib->pNext = p; - } - break; - } - } - } + u8 inTransaction; /* Transaction state */ + u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ + u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ + u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ + u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ + u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ + int nTransaction; /* Number of open transactions (read + write) */ + u32 nPage; /* Number of pages in the database */ + void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ + void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ + sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */ + Bitvec *pHasContent; /* Set of pages moved to free-list this transaction */ +#ifndef SQLITE_OMIT_SHARED_CACHE + int nRef; /* Number of references to this structure */ + BtShared *pNext; /* Next on a list of sharable BtShared structs */ + BtLock *pLock; /* List of locks held on this shared-btree struct */ + Btree *pWriter; /* Btree with currently open write transaction */ + u8 isExclusive; /* True if pWriter has an EXCLUSIVE lock on the db */ + u8 isPending; /* If waiting for read-locks to clear */ #endif - *ppBtree = p; + u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ +}; -btree_open_out: - if( rc!=SQLITE_OK ){ - if( pBt && pBt->pPager ){ - sqlite3PagerClose(pBt->pPager); - } - sqlite3_free(pBt); - sqlite3_free(p); - *ppBtree = 0; - }else{ - /* If the B-Tree was successfully opened, set the pager-cache size to the - ** default value. Except, when opening on an existing shared pager-cache, - ** do not change the pager-cache size. - */ - if( sqlite3BtreeSchema(p, 0, 0)==0 ){ - sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); - } - } - if( mutexOpen ){ - assert( sqlite3_mutex_held(mutexOpen) ); - sqlite3_mutex_leave(mutexOpen); - } - return rc; -} +/* +** An instance of the following structure is used to hold information +** about a cell. The parseCellPtr() function fills in this structure +** based on information extract from the raw disk page. +*/ +typedef struct CellInfo CellInfo; +struct CellInfo { + i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ + u8 *pCell; /* Pointer to the start of cell content */ + u32 nData; /* Number of bytes of data */ + u32 nPayload; /* Total amount of payload */ + u16 nHeader; /* Size of the cell content header in bytes */ + u16 nLocal; /* Amount of payload held locally */ + u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ + u16 nSize; /* Size of the cell content on the main b-tree page */ +}; /* -** Decrement the BtShared.nRef counter. When it reaches zero, -** remove the BtShared structure from the sharing list. Return -** true if the BtShared.nRef counter reaches zero and return -** false if it is still positive. +** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than +** this will be declared corrupt. This value is calculated based on a +** maximum database size of 2^31 pages a minimum fanout of 2 for a +** root-node and 3 for all other internal nodes. +** +** If a tree that appears to be taller than this is encountered, it is +** assumed that the database is corrupt. */ -static int removeFromSharingList(BtShared *pBt){ -#ifndef SQLITE_OMIT_SHARED_CACHE - sqlite3_mutex *pMaster; - BtShared *pList; - int removed = 0; +#define BTCURSOR_MAX_DEPTH 20 - assert( sqlite3_mutex_notheld(pBt->mutex) ); - pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); - sqlite3_mutex_enter(pMaster); - pBt->nRef--; - if( pBt->nRef<=0 ){ - if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ - GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt->pNext; - }else{ - pList = GLOBAL(BtShared*,sqlite3SharedCacheList); - while( ALWAYS(pList) && pList->pNext!=pBt ){ - pList=pList->pNext; - } - if( ALWAYS(pList) ){ - pList->pNext = pBt->pNext; - } - } - if( SQLITE_THREADSAFE ){ - sqlite3_mutex_free(pBt->mutex); - } - removed = 1; - } - sqlite3_mutex_leave(pMaster); - return removed; -#else - return 1; +/* +** A cursor is a pointer to a particular entry within a particular +** b-tree within a database file. +** +** The entry is identified by its MemPage and the index in +** MemPage.aCell[] of the entry. +** +** A single database file can shared by two more database connections, +** but cursors cannot be shared. Each cursor is associated with a +** particular database connection identified BtCursor.pBtree.db. +** +** Fields in this structure are accessed under the BtShared.mutex +** found at self->pBt->mutex. +*/ +struct BtCursor { + Btree *pBtree; /* The Btree to which this cursor belongs */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + Pgno pgnoRoot; /* The root page of this tree */ + sqlite3_int64 cachedRowid; /* Next rowid cache. 0 means not valid */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + void *pKey; /* Saved key that was cursor's last known position */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ + u8 wrFlag; /* True if writable */ + u8 atLast; /* Cursor pointing to the last entry */ + u8 validNKey; /* True if info.nKey is valid */ + u8 eState; /* One of the CURSOR_XXX constants (see below) */ +#ifndef SQLITE_OMIT_INCRBLOB + Pgno *aOverflow; /* Cache of overflow page locations */ + u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ #endif -} + i16 iPage; /* Index of current page in apPage */ + u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ + MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ +}; /* -** Make sure pBt->pTmpSpace points to an allocation of -** MX_CELL_SIZE(pBt) bytes. +** Potential values for BtCursor.eState. +** +** CURSOR_VALID: +** Cursor points to a valid entry. getPayload() etc. may be called. +** +** CURSOR_INVALID: +** Cursor does not point to a valid entry. This can happen (for example) +** because the table is empty or because BtreeCursorFirst() has not been +** called. +** +** CURSOR_REQUIRESEEK: +** The table that this cursor was opened on still exists, but has been +** modified since the cursor was last used. The cursor position is saved +** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in +** this state, restoreCursorPosition() can be called to attempt to +** seek the cursor to the saved position. +** +** CURSOR_FAULT: +** A unrecoverable error (an I/O error or a malloc failure) has occurred +** on a different connection that shares the BtShared cache with this +** cursor. The error has left the cache in an inconsistent state. +** Do nothing else with this cursor. Any attempt to use the cursor +** should return the error code stored in BtCursor.skip */ -static void allocateTempSpace(BtShared *pBt){ - if( !pBt->pTmpSpace ){ - pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); - } -} +#define CURSOR_INVALID 0 +#define CURSOR_VALID 1 +#define CURSOR_REQUIRESEEK 2 +#define CURSOR_FAULT 3 + +/* +** The database page the PENDING_BYTE occupies. This page is never used. +*/ +# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) /* -** Free the pBt->pTmpSpace allocation +** These macros define the location of the pointer-map entry for a +** database page. The first argument to each is the number of usable +** bytes on each page of the database (often 1024). The second is the +** page number to look up in the pointer map. +** +** PTRMAP_PAGENO returns the database page number of the pointer-map +** page that stores the required pointer. PTRMAP_PTROFFSET returns +** the offset of the requested map entry. +** +** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page, +** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be +** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements +** this test. */ -static void freeTempSpace(BtShared *pBt){ - sqlite3PageFree( pBt->pTmpSpace); - pBt->pTmpSpace = 0; -} +#define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno) +#define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1)) +#define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno)) /* -** Close an open database and invalidate all cursors. +** The pointer map is a lookup table that identifies the parent page for +** each child page in the database file. The parent page is the page that +** contains a pointer to the child. Every page in the database contains +** 0 or 1 parent pages. (In this context 'database page' refers +** to any page that is not part of the pointer map itself.) Each pointer map +** entry consists of a single byte 'type' and a 4 byte parent page number. +** The PTRMAP_XXX identifiers below are the valid types. +** +** The purpose of the pointer map is to facility moving pages from one +** position in the file to another as part of autovacuum. When a page +** is moved, the pointer in its parent must be updated to point to the +** new location. The pointer map is used to locate the parent page quickly. +** +** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not +** used in this case. +** +** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number +** is not used in this case. +** +** PTRMAP_OVERFLOW1: The database page is the first page in a list of +** overflow pages. The page number identifies the page that +** contains the cell with a pointer to this overflow page. +** +** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of +** overflow pages. The page-number identifies the previous +** page in the overflow page list. +** +** PTRMAP_BTREE: The database page is a non-root btree page. The page number +** identifies the parent page in the btree. */ -SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ - BtShared *pBt = p->pBt; - BtCursor *pCur; +#define PTRMAP_ROOTPAGE 1 +#define PTRMAP_FREEPAGE 2 +#define PTRMAP_OVERFLOW1 3 +#define PTRMAP_OVERFLOW2 4 +#define PTRMAP_BTREE 5 - /* Close all cursors opened via this handle. */ - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - pCur = pBt->pCursor; - while( pCur ){ - BtCursor *pTmp = pCur; - pCur = pCur->pNext; - if( pTmp->pBtree==p ){ - sqlite3BtreeCloseCursor(pTmp); - } - } +/* A bunch of assert() statements to check the transaction state variables +** of handle p (type Btree*) are internally consistent. +*/ +#define btreeIntegrity(p) \ + assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ + assert( p->pBt->inTransaction>=p->inTrans ); - /* Rollback any active transaction and free the handle structure. - ** The call to sqlite3BtreeRollback() drops any table-locks held by - ** this handle. - */ - sqlite3BtreeRollback(p); - sqlite3BtreeLeave(p); - /* If there are still other outstanding references to the shared-btree - ** structure, return now. The remainder of this procedure cleans - ** up the shared-btree. - */ - assert( p->wantToLock==0 && p->locked==0 ); - if( !p->sharable || removeFromSharingList(pBt) ){ - /* The pBt is no longer on the sharing list, so we can access - ** it without having to hold the mutex. - ** - ** Clean out and delete the BtShared object. - */ - assert( !pBt->pCursor ); - sqlite3PagerClose(pBt->pPager); - if( pBt->xFreeSchema && pBt->pSchema ){ - pBt->xFreeSchema(pBt->pSchema); - } - sqlite3DbFree(0, pBt->pSchema); - freeTempSpace(pBt); - sqlite3_free(pBt); - } +/* +** The ISAUTOVACUUM macro is used within balance_nonroot() to determine +** if the database supports auto-vacuum or not. Because it is used +** within an expression that is an argument to another macro +** (sqliteMallocRaw), it is not possible to use conditional compilation. +** So, this macro is defined instead. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define ISAUTOVACUUM (pBt->autoVacuum) +#else +#define ISAUTOVACUUM 0 +#endif + + +/* +** This structure is passed around through all the sanity checking routines +** in order to keep track of some global state information. +*/ +typedef struct IntegrityCk IntegrityCk; +struct IntegrityCk { + BtShared *pBt; /* The tree being checked out */ + Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ + Pgno nPage; /* Number of pages in the database */ + int *anRef; /* Number of times each page is referenced */ + int mxErr; /* Stop accumulating errors when this reaches zero */ + int nErr; /* Number of messages written to zErrMsg so far */ + int mallocFailed; /* A memory allocation error has occurred */ + StrAccum errMsg; /* Accumulate the error message text here */ +}; + +/* +** Read or write a two- and four-byte big-endian integer values. +*/ +#define get2byte(x) ((x)[0]<<8 | (x)[1]) +#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) +#define get4byte sqlite3Get4byte +#define put4byte sqlite3Put4byte +/************** End of btreeInt.h ********************************************/ +/************** Continuing where we left off in btmutex.c ********************/ #ifndef SQLITE_OMIT_SHARED_CACHE - assert( p->wantToLock==0 ); +#if SQLITE_THREADSAFE + +/* +** Obtain the BtShared mutex associated with B-Tree handle p. Also, +** set BtShared.db to the database handle associated with p and the +** p->locked boolean to true. +*/ +static void lockBtreeMutex(Btree *p){ assert( p->locked==0 ); - if( p->pPrev ) p->pPrev->pNext = p->pNext; - if( p->pNext ) p->pNext->pPrev = p->pPrev; -#endif + assert( sqlite3_mutex_notheld(p->pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3_free(p); - return SQLITE_OK; + sqlite3_mutex_enter(p->pBt->mutex); + p->pBt->db = p->db; + p->locked = 1; } /* -** Change the limit on the number of pages allowed in the cache. -** -** The maximum number of cache pages is set to the absolute -** value of mxPage. If mxPage is negative, the pager will -** operate asynchronously - it will not stop to do fsync()s -** to insure data is written to the disk surface before -** continuing. Transactions still work if synchronous is off, -** and the database cannot be corrupted if this program -** crashes. But if the operating system crashes or there is -** an abrupt power failure when synchronous is off, the database -** could be left in an inconsistent and unrecoverable state. -** Synchronous is on by default so database corruption is not -** normally a worry. +** Release the BtShared mutex associated with B-Tree handle p and +** clear the p->locked boolean. */ -SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ +static void unlockBtreeMutex(Btree *p){ BtShared *pBt = p->pBt; + assert( p->locked==1 ); + assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - sqlite3PagerSetCachesize(pBt->pPager, mxPage); - sqlite3BtreeLeave(p); - return SQLITE_OK; + assert( p->db==pBt->db ); + + sqlite3_mutex_leave(pBt->mutex); + p->locked = 0; } /* -** Change the way data is synced to disk in order to increase or decrease -** how well the database resists damage due to OS crashes and power -** failures. Level 1 is the same as asynchronous (no syncs() occur and -** there is a high probability of damage) Level 2 is the default. There -** is a very low but non-zero probability of damage. Level 3 reduces the -** probability of damage to near zero but with a write performance reduction. +** Enter a mutex on the given BTree object. +** +** If the object is not sharable, then no mutex is ever required +** and this routine is a no-op. The underlying mutex is non-recursive. +** But we keep a reference count in Btree.wantToLock so the behavior +** of this interface is recursive. +** +** To avoid deadlocks, multiple Btrees are locked in the same order +** by all database connections. The p->pNext is a list of other +** Btrees belonging to the same database connection as the p Btree +** which need to be locked after p. If we cannot get a lock on +** p, then first unlock all of the others on p->pNext, then wait +** for the lock to become available on p, then relock all of the +** subsequent Btrees that desire a lock. */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel( - Btree *p, /* The btree to set the safety level on */ - int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ - int fullSync, /* PRAGMA fullfsync. */ - int ckptFullSync /* PRAGMA checkpoint_fullfync */ -){ - BtShared *pBt = p->pBt; +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ + Btree *pLater; + + /* Some basic sanity checking on the Btree. The list of Btrees + ** connected by pNext and pPrev should be in sorted order by + ** Btree.pBt value. All elements of the list should belong to + ** the same connection. Only shared Btrees are on the list. */ + assert( p->pNext==0 || p->pNext->pBt>p->pBt ); + assert( p->pPrev==0 || p->pPrev->pBtpBt ); + assert( p->pNext==0 || p->pNext->db==p->db ); + assert( p->pPrev==0 || p->pPrev->db==p->db ); + assert( p->sharable || (p->pNext==0 && p->pPrev==0) ); + + /* Check for locking consistency */ + assert( !p->locked || p->wantToLock>0 ); + assert( p->sharable || p->wantToLock==0 ); + + /* We should already hold a lock on the database connection */ assert( sqlite3_mutex_held(p->db->mutex) ); - assert( level>=1 && level<=3 ); - sqlite3BtreeEnter(p); - sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync, ckptFullSync); - sqlite3BtreeLeave(p); - return SQLITE_OK; + + /* Unless the database is sharable and unlocked, then BtShared.db + ** should already be set correctly. */ + assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db ); + + if( !p->sharable ) return; + p->wantToLock++; + if( p->locked ) return; + + /* In most cases, we should be able to acquire the lock we + ** want without having to go throught the ascending lock + ** procedure that follows. Just be sure not to block. + */ + if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ + p->pBt->db = p->db; + p->locked = 1; + return; + } + + /* To avoid deadlock, first release all locks with a larger + ** BtShared address. Then acquire our lock. Then reacquire + ** the other BtShared locks that we used to hold in ascending + ** order. + */ + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + assert( pLater->sharable ); + assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt ); + assert( !pLater->locked || pLater->wantToLock>0 ); + if( pLater->locked ){ + unlockBtreeMutex(pLater); + } + } + lockBtreeMutex(p); + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + if( pLater->wantToLock ){ + lockBtreeMutex(pLater); + } + } } -#endif /* -** Return TRUE if the given btree is set to safety level 1. In other -** words, return TRUE if no sync() occurs on the disk files. +** Exit the recursive mutex on a Btree. */ -SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree *p){ - BtShared *pBt = p->pBt; - int rc; - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - assert( pBt && pBt->pPager ); - rc = sqlite3PagerNosync(pBt->pPager); - sqlite3BtreeLeave(p); - return rc; +SQLITE_PRIVATE void sqlite3BtreeLeave(Btree *p){ + if( p->sharable ){ + assert( p->wantToLock>0 ); + p->wantToLock--; + if( p->wantToLock==0 ){ + unlockBtreeMutex(p); + } + } } -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) +#ifndef NDEBUG /* -** Change the default pages size and the number of reserved bytes per page. -** Or, if the page size has already been fixed, return SQLITE_READONLY -** without changing anything. -** -** The page size must be a power of 2 between 512 and 65536. If the page -** size supplied does not meet this constraint then the page size is not -** changed. +** Return true if the BtShared mutex is held on the btree, or if the +** B-Tree is not marked as sharable. ** -** Page sizes are constrained to be a power of two so that the region -** of the database file used for locking (beginning at PENDING_BYTE, -** the first byte past the 1GB boundary, 0x40000000) needs to occur -** at the beginning of a page. +** This routine is used only from within assert() statements. +*/ +SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ + assert( p->sharable==0 || p->locked==0 || p->wantToLock>0 ); + assert( p->sharable==0 || p->locked==0 || p->db==p->pBt->db ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->pBt->mutex) ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->db->mutex) ); + + return (p->sharable==0 || p->locked); +} +#endif + + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter and leave a mutex on a Btree given a cursor owned by that +** Btree. These entry points are used by incremental I/O and can be +** omitted if that module is not used. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +#endif /* SQLITE_OMIT_INCRBLOB */ + + +/* +** Enter the mutex on every Btree associated with a database +** connection. This is needed (for example) prior to parsing +** a statement since we will be comparing table and column names +** against all schemas and we do not want those schemas being +** reset out from under us. ** -** If parameter nReserve is less than zero, then the number of reserved -** bytes per page is left unchanged. +** There is a corresponding leave-all procedures. ** -** If the iFix!=0 then the pageSizeFixed flag is set so that the page size -** and autovacuum mode can no longer be changed. +** Enter the mutexes in accending order by BtShared pointer address +** to avoid the possibility of deadlock when two threads with +** two or more btrees in common both try to lock all their btrees +** at the same instant. */ -SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ - int rc = SQLITE_OK; - BtShared *pBt = p->pBt; - assert( nReserve>=-1 && nReserve<=255 ); - sqlite3BtreeEnter(p); - if( pBt->pageSizeFixed ){ - sqlite3BtreeLeave(p); - return SQLITE_READONLY; - } - if( nReserve<0 ){ - nReserve = pBt->pageSize - pBt->usableSize; +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p ) sqlite3BtreeEnter(p); } - assert( nReserve>=0 && nReserve<=255 ); - if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && - ((pageSize-1)&pageSize)==0 ){ - assert( (pageSize & 7)==0 ); - assert( !pBt->pPage1 && !pBt->pCursor ); - pBt->pageSize = (u32)pageSize; - freeTempSpace(pBt); +} +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p ) sqlite3BtreeLeave(p); } - rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); - pBt->usableSize = pBt->pageSize - (u16)nReserve; - if( iFix ) pBt->pageSizeFixed = 1; - sqlite3BtreeLeave(p); - return rc; } /* -** Return the currently defined page size +** Return true if a particular Btree requires a lock. Return FALSE if +** no lock is ever required since it is not sharable. */ -SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ - return p->pBt->pageSize; +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ + return p->sharable; } +#ifndef NDEBUG /* -** Return the number of bytes of space at the end of every page that -** are intentually left unused. This is the "reserved" space that is -** sometimes used by extensions. +** Return true if the current thread holds the database connection +** mutex and all required BtShared mutexes. +** +** This routine is used inside assert() statements only. */ -SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree *p){ - int n; - sqlite3BtreeEnter(p); - n = p->pBt->pageSize - p->pBt->usableSize; - sqlite3BtreeLeave(p); - return n; +SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ + int i; + if( !sqlite3_mutex_held(db->mutex) ){ + return 0; + } + for(i=0; inDb; i++){ + Btree *p; + p = db->aDb[i].pBt; + if( p && p->sharable && + (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){ + return 0; + } + } + return 1; } +#endif /* NDEBUG */ +#ifndef NDEBUG /* -** Set the maximum page count for a database if mxPage is positive. -** No changes are made if mxPage is 0 or negative. -** Regardless of the value of mxPage, return the maximum page count. +** Return true if the correct mutexes are held for accessing the +** db->aDb[iDb].pSchema structure. The mutexes required for schema +** access are: +** +** (1) The mutex on db +** (2) if iDb!=1, then the mutex on db->aDb[iDb].pBt. +** +** If pSchema is not NULL, then iDb is computed from pSchema and +** db using sqlite3SchemaToIndex(). */ -SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ - int n; - sqlite3BtreeEnter(p); - n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); - sqlite3BtreeLeave(p); - return n; +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ + Btree *p; + assert( db!=0 ); + if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); + assert( iDb>=0 && iDbnDb ); + if( !sqlite3_mutex_held(db->mutex) ) return 0; + if( iDb==1 ) return 1; + p = db->aDb[iDb].pBt; + assert( p!=0 ); + return p->sharable==0 || p->locked==1; } +#endif /* NDEBUG */ +#else /* SQLITE_THREADSAFE>0 above. SQLITE_THREADSAFE==0 below */ /* -** Set the secureDelete flag if newFlag is 0 or 1. If newFlag is -1, -** then make no changes. Always return the value of the secureDelete -** setting after the change. +** The following are special cases for mutex enter routines for use +** in single threaded applications that use shared cache. Except for +** these two routines, all mutex operations are no-ops in that case and +** are null #defines in btree.h. +** +** If shared cache is disabled, then all btree mutex routines, including +** the ones below, are no-ops and are null #defines in btree.h. */ -SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ - int b; - if( p==0 ) return 0; - sqlite3BtreeEnter(p); - if( newFlag>=0 ){ - p->pBt->secureDelete = (newFlag!=0) ? 1 : 0; - } - b = p->pBt->secureDelete; - sqlite3BtreeLeave(p); - return b; + +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ + p->pBt->db = p->db; } -#endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + p->pBt->db = p->db; + } + } +} +#endif /* if SQLITE_THREADSAFE */ +#endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ + +/************** End of btmutex.c *********************************************/ +/************** Begin file btree.c *******************************************/ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a external (disk-based) database using BTrees. +** See the header comment on "btreeInt.h" for additional information. +** Including a description of file format and an overview of operation. +*/ /* -** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' -** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it -** is disabled. The default value for the auto-vacuum property is -** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. +** The header string that appears at the beginning of every +** SQLite database. */ -SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ -#ifdef SQLITE_OMIT_AUTOVACUUM - return SQLITE_READONLY; -#else - BtShared *pBt = p->pBt; - int rc = SQLITE_OK; - u8 av = (u8)autoVacuum; +static const char zMagicHeader[] = SQLITE_FILE_HEADER; - sqlite3BtreeEnter(p); - if( pBt->pageSizeFixed && (av ?1:0)!=pBt->autoVacuum ){ - rc = SQLITE_READONLY; - }else{ - pBt->autoVacuum = av ?1:0; - pBt->incrVacuum = av==2 ?1:0; - } - sqlite3BtreeLeave(p); - return rc; +/* +** Set this global variable to 1 to enable tracing using the TRACE +** macro. +*/ +#if 0 +int sqlite3BtreeTrace=1; /* True to enable tracing */ +# define TRACE(X) if(sqlite3BtreeTrace){printf X;fflush(stdout);} +#else +# define TRACE(X) #endif -} /* -** Return the value of the 'auto-vacuum' property. If auto-vacuum is -** enabled 1 is returned. Otherwise 0. +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. */ -SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ -#ifdef SQLITE_OMIT_AUTOVACUUM - return BTREE_AUTOVACUUM_NONE; +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** A list of BtShared objects that are eligible for participation +** in shared cache. This variable has file scope during normal builds, +** but the test harness needs to access it so we make it global for +** test builds. +** +** Access to this variable is protected by SQLITE_MUTEX_STATIC_MASTER. +*/ +#ifdef SQLITE_TEST +SQLITE_PRIVATE BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; #else - int rc; - sqlite3BtreeEnter(p); - rc = ( - (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: - (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: - BTREE_AUTOVACUUM_INCR - ); - sqlite3BtreeLeave(p); - return rc; +static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; #endif -} - +#endif /* SQLITE_OMIT_SHARED_CACHE */ +#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Get a reference to pPage1 of the database file. This will -** also acquire a readlock on that file. +** Enable or disable the shared pager and schema features. ** -** SQLITE_OK is returned on success. If the file is not a -** well-formed database file, then SQLITE_CORRUPT is returned. -** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM -** is returned if we run out of memory. +** This routine has no effect on existing database connections. +** The shared cache setting effects only future calls to +** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). */ -static int lockBtree(BtShared *pBt){ - int rc; /* Result code from subfunctions */ - MemPage *pPage1; /* Page 1 of the database file */ - int nPage; /* Number of pages in the database */ - int nPageFile = 0; /* Number of pages in the database file */ - int nPageHeader; /* Number of pages in the database according to hdr */ +SQLITE_API int sqlite3_enable_shared_cache(int enable){ + sqlite3GlobalConfig.sharedCacheEnabled = enable; + return SQLITE_OK; +} +#endif - assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pBt->pPage1==0 ); - rc = sqlite3PagerSharedLock(pBt->pPager); - if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); - if( rc!=SQLITE_OK ) return rc; - /* Do some checking to help insure the file we opened really is - ** a valid database file. + +#ifdef SQLITE_OMIT_SHARED_CACHE + /* + ** The functions querySharedCacheTableLock(), setSharedCacheTableLock(), + ** and clearAllSharedCacheTableLocks() + ** manipulate entries in the BtShared.pLock linked list used to store + ** shared-cache table level locks. If the library is compiled with the + ** shared-cache feature disabled, then there is only ever one user + ** of each BtShared structure and so this locking is not necessary. + ** So define the lock related functions as no-ops. */ - nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - sqlite3PagerPagecount(pBt->pPager, &nPageFile); - if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ - nPage = nPageFile; + #define querySharedCacheTableLock(a,b,c) SQLITE_OK + #define setSharedCacheTableLock(a,b,c) SQLITE_OK + #define clearAllSharedCacheTableLocks(a) + #define downgradeAllSharedCacheTableLocks(a) + #define hasSharedCacheTableLock(a,b,c,d) 1 + #define hasReadConflicts(a, b) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE + +#ifdef SQLITE_DEBUG +/* +**** This function is only used as part of an assert() statement. *** +** +** Check to see if pBtree holds the required locks to read or write to the +** table with root page iRoot. Return 1 if it does and 0 if not. +** +** For example, when writing to a table with root-page iRoot via +** Btree connection pBtree: +** +** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); +** +** When writing to an index that resides in a sharable database, the +** caller should have first obtained a lock specifying the root page of +** the corresponding table. This makes things a bit more complicated, +** as this module treats each table as a separate structure. To determine +** the table corresponding to the index being written, this +** function has to search through the database schema. +** +** Instead of a lock on the table/index rooted at page iRoot, the caller may +** hold a write-lock on the schema table (root page 1). This is also +** acceptable. +*/ +static int hasSharedCacheTableLock( + Btree *pBtree, /* Handle that must hold lock */ + Pgno iRoot, /* Root page of b-tree */ + int isIndex, /* True if iRoot is the root of an index b-tree */ + int eLockType /* Required lock type (READ_LOCK or WRITE_LOCK) */ +){ + Schema *pSchema = (Schema *)pBtree->pBt->pSchema; + Pgno iTab = 0; + BtLock *pLock; + + /* If this database is not shareable, or if the client is reading + ** and has the read-uncommitted flag set, then no lock is required. + ** Return true immediately. + */ + if( (pBtree->sharable==0) + || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted)) + ){ + return 1; } - if( nPage>0 ){ - u32 pageSize; - u32 usableSize; - u8 *page1 = pPage1->aData; - rc = SQLITE_NOTADB; - if( memcmp(page1, zMagicHeader, 16)!=0 ){ - goto page1_init_failed; - } -#ifdef SQLITE_OMIT_WAL - if( page1[18]>1 ){ - pBt->readOnly = 1; - } - if( page1[19]>1 ){ - goto page1_init_failed; - } -#else - if( page1[18]>2 ){ - pBt->readOnly = 1; - } - if( page1[19]>2 ){ - goto page1_init_failed; - } + /* If the client is reading or writing an index and the schema is + ** not loaded, then it is too difficult to actually check to see if + ** the correct locks are held. So do not bother - just return true. + ** This case does not come up very often anyhow. + */ + if( isIndex && (!pSchema || (pSchema->flags&DB_SchemaLoaded)==0) ){ + return 1; + } - /* If the write version is set to 2, this database should be accessed - ** in WAL mode. If the log is not already open, open it now. Then - ** return SQLITE_OK and return without populating BtShared.pPage1. - ** The caller detects this and calls this function again. This is - ** required as the version of page 1 currently in the page1 buffer - ** may not be the latest version - there may be a newer one in the log - ** file. - */ - if( page1[19]==2 && pBt->doNotUseWAL==0 ){ - int isOpen = 0; - rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); - if( rc!=SQLITE_OK ){ - goto page1_init_failed; - }else if( isOpen==0 ){ - releasePage(pPage1); - return SQLITE_OK; + /* Figure out the root-page that the lock should be held on. For table + ** b-trees, this is just the root page of the b-tree being read or + ** written. For index b-trees, it is the root page of the associated + ** table. */ + if( isIndex ){ + HashElem *p; + for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ + Index *pIdx = (Index *)sqliteHashData(p); + if( pIdx->tnum==(int)iRoot ){ + iTab = pIdx->pTable->tnum; } - rc = SQLITE_NOTADB; } -#endif + }else{ + iTab = iRoot; + } - /* The maximum embedded fraction must be exactly 25%. And the minimum - ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. - ** The original design allowed these amounts to vary, but as of - ** version 3.6.0, we require them to be fixed. - */ - if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ - goto page1_init_failed; - } - pageSize = (page1[16]<<8) | (page1[17]<<16); - if( ((pageSize-1)&pageSize)!=0 - || pageSize>SQLITE_MAX_PAGE_SIZE - || pageSize<=256 + /* Search for the required lock. Either a write-lock on root-page iTab, a + ** write-lock on the schema table, or (if the client is reading) a + ** read-lock on iTab will suffice. Return 1 if any of these are found. */ + for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ + if( pLock->pBtree==pBtree + && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) + && pLock->eLock>=eLockType ){ - goto page1_init_failed; - } - assert( (pageSize & 7)==0 ); - usableSize = pageSize - page1[20]; - if( (u32)pageSize!=pBt->pageSize ){ - /* After reading the first page of the database assuming a page size - ** of BtShared.pageSize, we have discovered that the page-size is - ** actually pageSize. Unlock the database, leave pBt->pPage1 at - ** zero and return SQLITE_OK. The caller will call this function - ** again with the correct page-size. - */ - releasePage(pPage1); - pBt->usableSize = usableSize; - pBt->pageSize = pageSize; - freeTempSpace(pBt); - rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, - pageSize-usableSize); - return rc; - } - if( nPageHeader>nPageFile ){ - rc = SQLITE_CORRUPT_BKPT; - goto page1_init_failed; - } - if( usableSize<480 ){ - goto page1_init_failed; + return 1; } - pBt->pageSize = pageSize; - pBt->usableSize = usableSize; -#ifndef SQLITE_OMIT_AUTOVACUUM - pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); - pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); -#endif } - /* maxLocal is the maximum amount of payload to store locally for - ** a cell. Make sure it is small enough so that at least minFanout - ** cells can will fit on one page. We assume a 10-byte page header. - ** Besides the payload, the cell must store: - ** 2-byte pointer to the cell - ** 4-byte child pointer - ** 9-byte nKey value - ** 4-byte nData value - ** 4-byte overflow page pointer - ** So a cell consists of a 2-byte pointer, a header which is as much as - ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow - ** page pointer. - */ - pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); - pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); - pBt->maxLeaf = (u16)(pBt->usableSize - 35); - pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); - assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); - pBt->pPage1 = pPage1; - pBt->nPage = nPage; - return SQLITE_OK; - -page1_init_failed: - releasePage(pPage1); - pBt->pPage1 = 0; - return rc; + /* Failed to find the required lock. */ + return 0; } +#endif /* SQLITE_DEBUG */ +#ifdef SQLITE_DEBUG /* -** If there are no outstanding cursors and we are not in the middle -** of a transaction but there is a read lock on the database, then -** this routine unrefs the first page of the database file which -** has the effect of releasing the read lock. +**** This function may be used as part of assert() statements only. **** ** -** If there is a transaction in progress, this routine is a no-op. +** Return true if it would be illegal for pBtree to write into the +** table or index rooted at iRoot because other shared connections are +** simultaneously reading that same table or index. +** +** It is illegal for pBtree to write if some other Btree object that +** shares the same BtShared object is currently reading or writing +** the iRoot table. Except, if the other Btree object has the +** read-uncommitted flag set, then it is OK for the other object to +** have a read cursor. +** +** For example, before writing to any part of the table or index +** rooted at page iRoot, one should call: +** +** assert( !hasReadConflicts(pBtree, iRoot) ); */ -static void unlockBtreeIfUnused(BtShared *pBt){ - assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pBt->pCursor==0 || pBt->inTransaction>TRANS_NONE ); - if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ - assert( pBt->pPage1->aData ); - assert( sqlite3PagerRefcount(pBt->pPager)==1 ); - assert( pBt->pPage1->aData ); - releasePage(pBt->pPage1); - pBt->pPage1 = 0; +static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ + BtCursor *p; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( p->pgnoRoot==iRoot + && p->pBtree!=pBtree + && 0==(p->pBtree->db->flags & SQLITE_ReadUncommitted) + ){ + return 1; + } } + return 0; } +#endif /* #ifdef SQLITE_DEBUG */ /* -** If pBt points to an empty file then convert that empty file -** into a new empty database by initializing the first page of -** the database. +** Query to see if Btree handle p may obtain a lock of type eLock +** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return +** SQLITE_OK if the lock may be obtained (by calling +** setSharedCacheTableLock()), or SQLITE_LOCKED if not. */ -static int newDatabase(BtShared *pBt){ - MemPage *pP1; - unsigned char *data; - int rc; +static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pIter; - assert( sqlite3_mutex_held(pBt->mutex) ); - if( pBt->nPage>0 ){ + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + assert( !(p->db->flags&SQLITE_ReadUncommitted)||eLock==WRITE_LOCK||iTab==1 ); + + /* If requesting a write-lock, then the Btree must have an open write + ** transaction on this file. And, obviously, for this to be so there + ** must be an open write transaction on the file itself. + */ + assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); + assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); + + /* This routine is a no-op if the shared-cache is not enabled */ + if( !p->sharable ){ return SQLITE_OK; } - pP1 = pBt->pPage1; - assert( pP1!=0 ); - data = pP1->aData; - rc = sqlite3PagerWrite(pP1->pDbPage); - if( rc ) return rc; - memcpy(data, zMagicHeader, sizeof(zMagicHeader)); - assert( sizeof(zMagicHeader)==16 ); - data[16] = (u8)((pBt->pageSize>>8)&0xff); - data[17] = (u8)((pBt->pageSize>>16)&0xff); - data[18] = 1; - data[19] = 1; - assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); - data[20] = (u8)(pBt->pageSize - pBt->usableSize); - data[21] = 64; - data[22] = 32; - data[23] = 32; - memset(&data[24], 0, 100-24); - zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); - pBt->pageSizeFixed = 1; -#ifndef SQLITE_OMIT_AUTOVACUUM - assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); - assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 ); - put4byte(&data[36 + 4*4], pBt->autoVacuum); - put4byte(&data[36 + 7*4], pBt->incrVacuum); -#endif - pBt->nPage = 1; - data[31] = 1; + + /* If some other connection is holding an exclusive lock, the + ** requested lock may not be obtained. + */ + if( pBt->pWriter!=p && pBt->isExclusive ){ + sqlite3ConnectionBlocked(p->db, pBt->pWriter->db); + return SQLITE_LOCKED_SHAREDCACHE; + } + + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + /* The condition (pIter->eLock!=eLock) in the following if(...) + ** statement is a simplification of: + ** + ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) + ** + ** since we know that if eLock==WRITE_LOCK, then no other connection + ** may hold a WRITE_LOCK on any table in this file (since there can + ** only be a single writer). + */ + assert( pIter->eLock==READ_LOCK || pIter->eLock==WRITE_LOCK ); + assert( eLock==READ_LOCK || pIter->pBtree==p || pIter->eLock==READ_LOCK); + if( pIter->pBtree!=p && pIter->iTable==iTab && pIter->eLock!=eLock ){ + sqlite3ConnectionBlocked(p->db, pIter->pBtree->db); + if( eLock==WRITE_LOCK ){ + assert( p==pBt->pWriter ); + pBt->isPending = 1; + } + return SQLITE_LOCKED_SHAREDCACHE; + } + } return SQLITE_OK; } +#endif /* !SQLITE_OMIT_SHARED_CACHE */ +#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Attempt to start a new transaction. A write-transaction -** is started if the second argument is nonzero, otherwise a read- -** transaction. If the second argument is 2 or more and exclusive -** transaction is started, meaning that no other process is allowed -** to access the database. A preexisting transaction may not be -** upgraded to exclusive by calling this routine a second time - the -** exclusivity flag only works for a new transaction. +** Add a lock on the table with root-page iTable to the shared-btree used +** by Btree handle p. Parameter eLock must be either READ_LOCK or +** WRITE_LOCK. ** -** A write-transaction must be started before attempting any -** changes to the database. None of the following routines -** will work unless a transaction is started first: +** This function assumes the following: ** -** sqlite3BtreeCreateTable() -** sqlite3BtreeCreateIndex() -** sqlite3BtreeClearTable() -** sqlite3BtreeDropTable() -** sqlite3BtreeInsert() -** sqlite3BtreeDelete() -** sqlite3BtreeUpdateMeta() +** (a) The specified Btree object p is connected to a sharable +** database (one with the BtShared.sharable flag set), and ** -** If an initial attempt to acquire the lock fails because of lock contention -** and the database was previously unlocked, then invoke the busy handler -** if there is one. But if there was previously a read-lock, do not -** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is -** returned when there is already a read-lock in order to avoid a deadlock. +** (b) No other Btree objects hold a lock that conflicts +** with the requested lock (i.e. querySharedCacheTableLock() has +** already been called and returned SQLITE_OK). ** -** Suppose there are two processes A and B. A has a read lock and B has -** a reserved lock. B tries to promote to exclusive but is blocked because -** of A's read lock. A tries to promote to reserved but is blocked by B. -** One or the other of the two processes must give way or there can be -** no progress. By returning SQLITE_BUSY and not invoking the busy callback -** when A already has a read lock, we encourage A to give up and let B -** proceed. +** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM +** is returned if a malloc attempt fails. */ -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ - sqlite3 *pBlock = 0; +static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtShared *pBt = p->pBt; - int rc = SQLITE_OK; + BtLock *pLock = 0; + BtLock *pIter; - sqlite3BtreeEnter(p); - btreeIntegrity(p); + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); - /* If the btree is already in a write-transaction, or it - ** is already in a read-transaction and a read-transaction - ** is requested, this is a no-op. - */ - if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ - goto trans_begun; - } + /* A connection with the read-uncommitted flag set will never try to + ** obtain a read-lock using this function. The only read-lock obtained + ** by a connection in read-uncommitted mode is on the sqlite_master + ** table, and that lock is obtained in BtreeBeginTrans(). */ + assert( 0==(p->db->flags&SQLITE_ReadUncommitted) || eLock==WRITE_LOCK ); - /* Write transactions are not possible on a read-only database */ - if( pBt->readOnly && wrflag ){ - rc = SQLITE_READONLY; - goto trans_begun; + /* This function should only be called on a sharable b-tree after it + ** has been determined that no other b-tree holds a conflicting lock. */ + assert( p->sharable ); + assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); + + /* First search the list for an existing lock on this table. */ + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->iTable==iTable && pIter->pBtree==p ){ + pLock = pIter; + break; + } } -#ifndef SQLITE_OMIT_SHARED_CACHE - /* If another database handle has already opened a write transaction - ** on this shared-btree structure and a second write transaction is - ** requested, return SQLITE_LOCKED. + /* If the above search did not find a BtLock struct associating Btree p + ** with table iTable, allocate one and link it into the list. */ - if( (wrflag && pBt->inTransaction==TRANS_WRITE) || pBt->isPending ){ - pBlock = pBt->pWriter->db; - }else if( wrflag>1 ){ - BtLock *pIter; - for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - if( pIter->pBtree!=p ){ - pBlock = pIter->pBtree->db; - break; - } + if( !pLock ){ + pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); + if( !pLock ){ + return SQLITE_NOMEM; } + pLock->iTable = iTable; + pLock->pBtree = p; + pLock->pNext = pBt->pLock; + pBt->pLock = pLock; } - if( pBlock ){ - sqlite3ConnectionBlocked(p->db, pBlock); - rc = SQLITE_LOCKED_SHAREDCACHE; - goto trans_begun; + + /* Set the BtLock.eLock variable to the maximum of the current lock + ** and the requested lock. This means if a write-lock was already held + ** and a read-lock requested, we don't incorrectly downgrade the lock. + */ + assert( WRITE_LOCK>READ_LOCK ); + if( eLock>pLock->eLock ){ + pLock->eLock = eLock; } -#endif - /* Any read-only or read-write transaction implies a read-lock on - ** page 1. So if some other shared-cache client already has a write-lock - ** on page 1, the transaction cannot be opened. */ - rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); - if( SQLITE_OK!=rc ) goto trans_begun; + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ - pBt->initiallyEmpty = (u8)(pBt->nPage==0); - do { - /* Call lockBtree() until either pBt->pPage1 is populated or - ** lockBtree() returns something other than SQLITE_OK. lockBtree() - ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after - ** reading page 1 it discovers that the page-size of the database - ** file is not pBt->pageSize. In this case lockBtree() will update - ** pBt->pageSize to the page-size of the file on disk. - */ - while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Release all the table locks (locks obtained via calls to +** the setSharedCacheTableLock() procedure) held by Btree object p. +** +** This function assumes that Btree p has an open read or write +** transaction. If it does not, then the BtShared.isPending variable +** may be incorrectly cleared. +*/ +static void clearAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + BtLock **ppIter = &pBt->pLock; - if( rc==SQLITE_OK && wrflag ){ - if( pBt->readOnly ){ - rc = SQLITE_READONLY; - }else{ - rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); - if( rc==SQLITE_OK ){ - rc = newDatabase(pBt); - } - } - } - - if( rc!=SQLITE_OK ){ - unlockBtreeIfUnused(pBt); - } - }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && - btreeInvokeBusyHandler(pBt) ); + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->sharable || 0==*ppIter ); + assert( p->inTrans>0 ); - if( rc==SQLITE_OK ){ - if( p->inTrans==TRANS_NONE ){ - pBt->nTransaction++; -#ifndef SQLITE_OMIT_SHARED_CACHE - if( p->sharable ){ - assert( p->lock.pBtree==p && p->lock.iTable==1 ); - p->lock.eLock = READ_LOCK; - p->lock.pNext = pBt->pLock; - pBt->pLock = &p->lock; - } -#endif - } - p->inTrans = (wrflag?TRANS_WRITE:TRANS_READ); - if( p->inTrans>pBt->inTransaction ){ - pBt->inTransaction = p->inTrans; - } - if( wrflag ){ - MemPage *pPage1 = pBt->pPage1; -#ifndef SQLITE_OMIT_SHARED_CACHE - assert( !pBt->pWriter ); - pBt->pWriter = p; - pBt->isExclusive = (u8)(wrflag>1); -#endif - - /* If the db-size header field is incorrect (as it may be if an old - ** client has been writing the database file), update it now. Doing - ** this sooner rather than later means the database size can safely - ** re-read the database size from page 1 if a savepoint or transaction - ** rollback occurs within the transaction. - */ - if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ - rc = sqlite3PagerWrite(pPage1->pDbPage); - if( rc==SQLITE_OK ){ - put4byte(&pPage1->aData[28], pBt->nPage); - } + while( *ppIter ){ + BtLock *pLock = *ppIter; + assert( pBt->isExclusive==0 || pBt->pWriter==pLock->pBtree ); + assert( pLock->pBtree->inTrans>=pLock->eLock ); + if( pLock->pBtree==p ){ + *ppIter = pLock->pNext; + assert( pLock->iTable!=1 || pLock==&p->lock ); + if( pLock->iTable!=1 ){ + sqlite3_free(pLock); } + }else{ + ppIter = &pLock->pNext; } } - -trans_begun: - if( rc==SQLITE_OK && wrflag ){ - /* This call makes sure that the pager has the correct number of - ** open savepoints. If the second parameter is greater than 0 and - ** the sub-journal is not already open, then it will be opened here. + assert( pBt->isPending==0 || pBt->pWriter ); + if( pBt->pWriter==p ){ + pBt->pWriter = 0; + pBt->isExclusive = 0; + pBt->isPending = 0; + }else if( pBt->nTransaction==2 ){ + /* This function is called when Btree p is concluding its + ** transaction. If there currently exists a writer, and p is not + ** that writer, then the number of locks held by connections other + ** than the writer must be about to drop to zero. In this case + ** set the isPending flag to 0. + ** + ** If there is not currently a writer, then BtShared.isPending must + ** be zero already. So this next line is harmless in that case. */ - rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + pBt->isPending = 0; } - - btreeIntegrity(p); - sqlite3BtreeLeave(p); - return rc; } -#ifndef SQLITE_OMIT_AUTOVACUUM - /* -** Set the pointer-map entries for all children of page pPage. Also, if -** pPage contains cells that point to overflow pages, set the pointer -** map entries for the overflow pages as well. +** This function changes all write-locks held by Btree p into read-locks. */ -static int setChildPtrmaps(MemPage *pPage){ - int i; /* Counter variable */ - int nCell; /* Number of cells in page pPage */ - int rc; /* Return code */ - BtShared *pBt = pPage->pBt; - u8 isInitOrig = pPage->isInit; - Pgno pgno = pPage->pgno; - - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - rc = btreeInitPage(pPage); - if( rc!=SQLITE_OK ){ - goto set_child_ptrmaps_out; +static void downgradeAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->pWriter==p ){ + BtLock *pLock; + pBt->pWriter = 0; + pBt->isExclusive = 0; + pBt->isPending = 0; + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + assert( pLock->eLock==READ_LOCK || pLock->pBtree==p ); + pLock->eLock = READ_LOCK; + } } - nCell = pPage->nCell; +} - for(i=0; ileaf ){ - Pgno childPgno = get4byte(pCell); - ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); - } - } +/* +***** This routine is used inside of assert() only **** +** +** Verify that the cursor holds the mutex on its BtShared +*/ +#ifdef SQLITE_DEBUG +static int cursorHoldsMutex(BtCursor *p){ + return sqlite3_mutex_held(p->pBt->mutex); +} +#endif - if( !pPage->leaf ){ - Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); - } -set_child_ptrmaps_out: - pPage->isInit = isInitOrig; - return rc; +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Invalidate the overflow page-list cache for cursor pCur, if any. +*/ +static void invalidateOverflowCache(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->aOverflow); + pCur->aOverflow = 0; } /* -** Somewhere on pPage is a pointer to page iFrom. Modify this pointer so -** that it points to iTo. Parameter eType describes the type of pointer to -** be modified, as follows: -** -** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child -** page of pPage. +** Invalidate the overflow page-list cache for all cursors opened +** on the shared btree structure pBt. +*/ +static void invalidateAllOverflowCache(BtShared *pBt){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + for(p=pBt->pCursor; p; p=p->pNext){ + invalidateOverflowCache(p); + } +} + +/* +** This function is called before modifying the contents of a table +** to invalidate any incrblob cursors that are open on the +** row or one of the rows being modified. ** -** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow -** page pointed to by one of the cells on pPage. +** If argument isClearTable is true, then the entire contents of the +** table is about to be deleted. In this case invalidate all incrblob +** cursors open on any row within the table with root-page pgnoRoot. ** -** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next -** overflow page in the list. +** Otherwise, if argument isClearTable is false, then the row with +** rowid iRow is being replaced or deleted. In this case invalidate +** only those incrblob cursors open on that specific row. */ -static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( eType==PTRMAP_OVERFLOW2 ){ - /* The pointer is always the first 4 bytes of the page in this case. */ - if( get4byte(pPage->aData)!=iFrom ){ - return SQLITE_CORRUPT_BKPT; +static void invalidateIncrblobCursors( + Btree *pBtree, /* The database file to check */ + i64 iRow, /* The rowid that might be changing */ + int isClearTable /* True if all rows are being deleted */ +){ + BtCursor *p; + BtShared *pBt = pBtree->pBt; + assert( sqlite3BtreeHoldsMutex(pBtree) ); + for(p=pBt->pCursor; p; p=p->pNext){ + if( p->isIncrblobHandle && (isClearTable || p->info.nKey==iRow) ){ + p->eState = CURSOR_INVALID; } - put4byte(pPage->aData, iTo); - }else{ - u8 isInitOrig = pPage->isInit; - int i; - int nCell; + } +} - btreeInitPage(pPage); - nCell = pPage->nCell; +#else + /* Stub functions when INCRBLOB is omitted */ + #define invalidateOverflowCache(x) + #define invalidateAllOverflowCache(x) + #define invalidateIncrblobCursors(x,y,z) +#endif /* SQLITE_OMIT_INCRBLOB */ - for(i=0; iaData[pPage->hdrOffset+8])!=iFrom ){ - return SQLITE_CORRUPT_BKPT; - } - put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); +/* +** Set bit pgno of the BtShared.pHasContent bitvec. This is called +** when a page that previously contained data becomes a free-list leaf +** page. +** +** The BtShared.pHasContent bitvec exists to work around an obscure +** bug caused by the interaction of two useful IO optimizations surrounding +** free-list leaf pages: +** +** 1) When all data is deleted from a page and the page becomes +** a free-list leaf page, the page is not written to the database +** (as free-list leaf pages contain no meaningful data). Sometimes +** such a page is not even journalled (as it will not be modified, +** why bother journalling it?). +** +** 2) When a free-list leaf page is reused, its content is not read +** from the database or written to the journal file (why should it +** be, if it is not at all meaningful?). +** +** By themselves, these optimizations work fine and provide a handy +** performance boost to bulk delete or insert operations. However, if +** a page is moved to the free-list and then reused within the same +** transaction, a problem comes up. If the page is not journalled when +** it is moved to the free-list and it is also not journalled when it +** is extracted from the free-list and reused, then the original data +** may be lost. In the event of a rollback, it may not be possible +** to restore the database to its original configuration. +** +** The solution is the BtShared.pHasContent bitvec. Whenever a page is +** moved to become a free-list leaf page, the corresponding bit is +** set in the bitvec. Whenever a leaf page is extracted from the free-list, +** optimization 2 above is omitted if the corresponding bit is already +** set in BtShared.pHasContent. The contents of the bitvec are cleared +** at the end of every transaction. +*/ +static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ + int rc = SQLITE_OK; + if( !pBt->pHasContent ){ + assert( pgno<=pBt->nPage ); + pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); + if( !pBt->pHasContent ){ + rc = SQLITE_NOMEM; } - - pPage->isInit = isInitOrig; } - return SQLITE_OK; + if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ + rc = sqlite3BitvecSet(pBt->pHasContent, pgno); + } + return rc; +} + +/* +** Query the BtShared.pHasContent vector. +** +** This function is called when a free-list leaf page is removed from the +** free-list for reuse. It returns false if it is safe to retrieve the +** page from the pager layer with the 'no-content' flag set. True otherwise. +*/ +static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ + Bitvec *p = pBt->pHasContent; + return (p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTest(p, pgno))); } +/* +** Clear (destroy) the BtShared.pHasContent bitvec. This should be +** invoked at the conclusion of each write-transaction. +*/ +static void btreeClearHasContent(BtShared *pBt){ + sqlite3BitvecDestroy(pBt->pHasContent); + pBt->pHasContent = 0; +} /* -** Move the open database page pDbPage to location iFreePage in the -** database. The pDbPage reference remains valid. +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. ** -** The isCommit flag indicates that there is no need to remember that -** the journal needs to be sync()ed before database page pDbPage->pgno -** can be written to. The caller has already promised not to write to that -** page. +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. */ -static int relocatePage( - BtShared *pBt, /* Btree */ - MemPage *pDbPage, /* Open page to move */ - u8 eType, /* Pointer map 'type' entry for pDbPage */ - Pgno iPtrPage, /* Pointer map 'page-no' entry for pDbPage */ - Pgno iFreePage, /* The location to move pDbPage to */ - int isCommit /* isCommit flag passed to sqlite3PagerMovepage */ -){ - MemPage *pPtrPage; /* The page that contains a pointer to pDbPage */ - Pgno iDbPage = pDbPage->pgno; - Pager *pPager = pBt->pPager; +static int saveCursorPosition(BtCursor *pCur){ int rc; - assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || - eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); - assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pDbPage->pBt==pBt ); + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); - /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", - iDbPage, iFreePage, iPtrPage, eType)); - rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); - if( rc!=SQLITE_OK ){ - return rc; - } - pDbPage->pgno = iFreePage; + rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); + assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - /* If pDbPage was a btree-page, then it may have child pages and/or cells - ** that point to overflow pages. The pointer map entries for all these - ** pages need to be changed. - ** - ** If pDbPage is an overflow page, then the first 4 bytes may store a - ** pointer to a subsequent overflow page. If this is the case, then - ** the pointer map needs to be updated for the subsequent overflow page. + /* If this is an intKey table, then the above call to BtreeKeySize() + ** stores the integer key in pCur->nKey. In this case this value is + ** all that is required. Otherwise, if pCur is not open on an intKey + ** table, then malloc space for and store the pCur->nKey bytes of key + ** data. */ - if( eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ){ - rc = setChildPtrmaps(pDbPage); - if( rc!=SQLITE_OK ){ - return rc; - } - }else{ - Pgno nextOvfl = get4byte(pDbPage->aData); - if( nextOvfl!=0 ){ - ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage, &rc); - if( rc!=SQLITE_OK ){ - return rc; + if( 0==pCur->apPage[0]->intKey ){ + void *pKey = sqlite3Malloc( (int)pCur->nKey ); + if( pKey ){ + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); } + }else{ + rc = SQLITE_NOMEM; } } + assert( !pCur->apPage[0]->intKey || !pCur->pKey ); - /* Fix the database pointer on page iPtrPage that pointed at iDbPage so - ** that it points at iFreePage. Also fix the pointer map entry for - ** iPtrPage. - */ - if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pPtrPage->pDbPage); - if( rc!=SQLITE_OK ){ - releasePage(pPtrPage); - return rc; - } - rc = modifyPagePointer(pPtrPage, iDbPage, iFreePage, eType); - releasePage(pPtrPage); - if( rc==SQLITE_OK ){ - ptrmapPut(pBt, iFreePage, eType, iPtrPage, &rc); + if( rc==SQLITE_OK ){ + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; } + pCur->iPage = -1; + pCur->eState = CURSOR_REQUIRESEEK; } + + invalidateOverflowCache(pCur); return rc; } -/* Forward declaration required by incrVacuumStep(). */ -static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); - /* -** Perform a single step of an incremental-vacuum. If successful, -** return SQLITE_OK. If there is no work to do (and therefore no -** point in calling this function again), return SQLITE_DONE. -** -** More specificly, this function attempts to re-organize the -** database so that the last page of the file currently in use -** is no longer in use. -** -** If the nFin parameter is non-zero, this function assumes -** that the caller will keep calling incrVacuumStep() until -** it returns SQLITE_DONE or an error, and that nFin is the -** number of pages the database file will contain after this -** process is complete. If nFin is zero, it is assumed that -** incrVacuumStep() will be called a finite amount of times -** which may or may not empty the freelist. A full autovacuum -** has nFin>0. A "PRAGMA incremental_vacuum" has nFin==0. +** Save the positions of all cursors (except pExcept) that are open on +** the table with root-page iRoot. Usually, this is called just before cursor +** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()). */ -static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ - Pgno nFreeList; /* Number of pages still on the free-list */ - int rc; - +static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ + BtCursor *p; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( iLastPg>nFin ); - - if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ - u8 eType; - Pgno iPtrPage; - - nFreeList = get4byte(&pBt->pPage1->aData[36]); - if( nFreeList==0 ){ - return SQLITE_DONE; - } - - rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage); - if( rc!=SQLITE_OK ){ - return rc; - } - if( eType==PTRMAP_ROOTPAGE ){ - return SQLITE_CORRUPT_BKPT; - } - - if( eType==PTRMAP_FREEPAGE ){ - if( nFin==0 ){ - /* Remove the page from the files free-list. This is not required - ** if nFin is non-zero. In that case, the free-list will be - ** truncated to zero after this function returns, so it doesn't - ** matter if it still contains some garbage entries. - */ - Pgno iFreePg; - MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( iFreePg==iLastPg ); - releasePage(pFreePg); - } - } else { - Pgno iFreePg; /* Index of free page to move pLastPg to */ - MemPage *pLastPg; - - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* If nFin is zero, this loop runs exactly once and page pLastPg - ** is swapped with the first free page pulled off the free list. - ** - ** On the other hand, if nFin is greater than zero, then keep - ** looping until a free-page located within the first nFin pages - ** of the file is found. - */ - do { - MemPage *pFreePg; - rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); - if( rc!=SQLITE_OK ){ - releasePage(pLastPg); - return rc; - } - releasePage(pFreePg); - }while( nFin!=0 && iFreePg>nFin ); - assert( iFreePgpDbPage); - if( rc==SQLITE_OK ){ - rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); - } - releasePage(pLastPg); - if( rc!=SQLITE_OK ){ + assert( pExcept==0 || pExcept->pBt==pBt ); + for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && + p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ return rc; } } } - - if( nFin==0 ){ - iLastPg--; - while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ - if( PTRMAP_ISPAGE(pBt, iLastPg) ){ - MemPage *pPg; - rc = btreeGetPage(pBt, iLastPg, &pPg, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pPg->pDbPage); - releasePage(pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - } - iLastPg--; - } - sqlite3PagerTruncateImage(pBt->pPager, iLastPg); - pBt->nPage = iLastPg; - } return SQLITE_OK; } /* -** A write-transaction must be opened before calling this function. -** It performs a single unit of work towards an incremental vacuum. -** -** If the incremental vacuum is finished after this function has run, -** SQLITE_DONE is returned. If it is not finished, but no error occurred, -** SQLITE_OK is returned. Otherwise an SQLite error code. +** Clear the current cursor position. */ -SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ - int rc; - BtShared *pBt = p->pBt; +SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + pCur->eState = CURSOR_INVALID; +} - sqlite3BtreeEnter(p); - assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); - if( !pBt->autoVacuum ){ - rc = SQLITE_DONE; +/* +** In this version of BtreeMoveto, pKey is a packed index record +** such as is generated by the OP_MakeRecord opcode. Unpack the +** record and then call BtreeMovetoUnpacked() to do the work. +*/ +static int btreeMoveto( + BtCursor *pCur, /* Cursor open on the btree to be searched */ + const void *pKey, /* Packed key if the btree is an index */ + i64 nKey, /* Integer key for tables. Size of pKey for indices */ + int bias, /* Bias search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; /* Status code */ + UnpackedRecord *pIdxKey; /* Unpacked index key */ + char aSpace[150]; /* Temp space for pIdxKey - to avoid a malloc */ + + if( pKey ){ + assert( nKey==(i64)(int)nKey ); + pIdxKey = sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, + aSpace, sizeof(aSpace)); + if( pIdxKey==0 ) return SQLITE_NOMEM; }else{ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[28], pBt->nPage); - } + pIdxKey = 0; + } + rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); + if( pKey ){ + sqlite3VdbeDeleteUnpackedRecord(pIdxKey); } - sqlite3BtreeLeave(p); return rc; } /* -** This routine is called prior to sqlite3PagerCommit when a transaction -** is commited for an auto-vacuum database. -** -** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages -** the database file should be truncated to during the commit process. -** i.e. the database has been reorganized so that only the first *pnTrunc -** pages are in use. +** Restore the cursor to the position it was in (or as close to as possible) +** when saveCursorPosition() was called. Note that this call deletes the +** saved position info stored by saveCursorPosition(), so there can be +** at most one effective restoreCursorPosition() call after each +** saveCursorPosition(). */ -static int autoVacuumCommit(BtShared *pBt){ - int rc = SQLITE_OK; - Pager *pPager = pBt->pPager; - VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); - - assert( sqlite3_mutex_held(pBt->mutex) ); - invalidateAllOverflowCache(pBt); - assert(pBt->autoVacuum); - if( !pBt->incrVacuum ){ - Pgno nFin; /* Number of pages in database after autovacuuming */ - Pgno nFree; /* Number of pages on the freelist initially */ - Pgno nPtrmap; /* Number of PtrMap pages to be freed */ - Pgno iFree; /* The next page to be freed */ - int nEntry; /* Number of entries on one ptrmap page */ - Pgno nOrig; /* Database size before freeing */ - - nOrig = btreePagecount(pBt); - if( PTRMAP_ISPAGE(pBt, nOrig) || nOrig==PENDING_BYTE_PAGE(pBt) ){ - /* It is not possible to create a database for which the final page - ** is either a pointer-map page or the pending-byte page. If one - ** is encountered, this indicates corruption. - */ - return SQLITE_CORRUPT_BKPT; - } - - nFree = get4byte(&pBt->pPage1->aData[36]); - nEntry = pBt->usableSize/5; - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinnOrig ) return SQLITE_CORRUPT_BKPT; - - for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ - rc = incrVacuumStep(pBt, nFin, iFree); - } - if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[32], 0); - put4byte(&pBt->pPage1->aData[36], 0); - put4byte(&pBt->pPage1->aData[28], nFin); - sqlite3PagerTruncateImage(pBt->pPager, nFin); - pBt->nPage = nFin; - } - if( rc!=SQLITE_OK ){ - sqlite3PagerRollback(pPager); - } +static int btreeRestoreCursorPosition(BtCursor *pCur){ + int rc; + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState>=CURSOR_REQUIRESEEK ); + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skipNext; + } + pCur->eState = CURSOR_INVALID; + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skipNext); + if( rc==SQLITE_OK ){ + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); } - - assert( nRef==sqlite3PagerRefcount(pPager) ); return rc; } -#else /* ifndef SQLITE_OMIT_AUTOVACUUM */ -# define setChildPtrmaps(x) SQLITE_OK -#endif +#define restoreCursorPosition(p) \ + (p->eState>=CURSOR_REQUIRESEEK ? \ + btreeRestoreCursorPosition(p) : \ + SQLITE_OK) /* -** This routine does the first phase of a two-phase commit. This routine -** causes a rollback journal to be created (if it does not already exist) -** and populated with enough information so that if a power loss occurs -** the database can be restored to its original state by playing back -** the journal. Then the contents of the journal are flushed out to -** the disk. After the journal is safely on oxide, the changes to the -** database are written into the database file and flushed to oxide. -** At the end of this call, the rollback journal still exists on the -** disk and we are still holding all locks, so the transaction has not -** committed. See sqlite3BtreeCommitPhaseTwo() for the second phase of the -** commit process. -** -** This call is a no-op if no write-transaction is currently active on pBt. -** -** Otherwise, sync the database file for the btree pBt. zMaster points to -** the name of a master journal file that should be written into the -** individual journal file, or is NULL, indicating no master journal file -** (single database transaction). -** -** When this is called, the master journal should already have been -** created, populated with this journal pointer and synced to disk. +** Determine whether or not a cursor has moved from the position it +** was last placed at. Cursors can move when the row they are pointing +** at is deleted out from under them. ** -** Once this is routine has returned, the only thing required to commit -** the write-transaction for this database file is to delete the journal. +** This routine returns an error code if something goes wrong. The +** integer *pHasMoved is set to one if the cursor has moved and 0 if not. */ -SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ - int rc = SQLITE_OK; - if( p->inTrans==TRANS_WRITE ){ - BtShared *pBt = p->pBt; - sqlite3BtreeEnter(p); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - rc = autoVacuumCommit(pBt); - if( rc!=SQLITE_OK ){ - sqlite3BtreeLeave(p); - return rc; - } - } -#endif - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); - sqlite3BtreeLeave(p); +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ + int rc; + + rc = restoreCursorPosition(pCur); + if( rc ){ + *pHasMoved = 1; + return rc; } - return rc; + if( pCur->eState!=CURSOR_VALID || pCur->skipNext!=0 ){ + *pHasMoved = 1; + }else{ + *pHasMoved = 0; + } + return SQLITE_OK; } +#ifndef SQLITE_OMIT_AUTOVACUUM /* -** This function is called from both BtreeCommitPhaseTwo() and BtreeRollback() -** at the conclusion of a transaction. +** Given a page number of a regular database page, return the page +** number for the pointer-map page that contains the entry for the +** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. */ -static void btreeEndTransaction(Btree *p){ - BtShared *pBt = p->pBt; - assert( sqlite3BtreeHoldsMutex(p) ); - - btreeClearHasContent(pBt); - if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ - /* If there are other active statements that belong to this database - ** handle, downgrade to a read-only transaction. The other statements - ** may still be reading from the database. */ - downgradeAllSharedCacheTableLocks(p); - p->inTrans = TRANS_READ; - }else{ - /* If the handle had any kind of transaction open, decrement the - ** transaction count of the shared btree. If the transaction count - ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() - ** call below will unlock the pager. */ - if( p->inTrans!=TRANS_NONE ){ - clearAllSharedCacheTableLocks(p); - pBt->nTransaction--; - if( 0==pBt->nTransaction ){ - pBt->inTransaction = TRANS_NONE; - } - } - - /* Set the current transaction state to TRANS_NONE and unlock the - ** pager if this call closed the only read or write transaction. */ - p->inTrans = TRANS_NONE; - unlockBtreeIfUnused(pBt); +static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ + int nPagesPerMapPage; + Pgno iPtrMap, ret; + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; + nPagesPerMapPage = (pBt->usableSize/5)+1; + iPtrMap = (pgno-2)/nPagesPerMapPage; + ret = (iPtrMap*nPagesPerMapPage) + 2; + if( ret==PENDING_BYTE_PAGE(pBt) ){ + ret++; } - - btreeIntegrity(p); + return ret; } /* -** Commit the transaction currently in progress. +** Write an entry into the pointer map. ** -** This routine implements the second phase of a 2-phase commit. The -** sqlite3BtreeCommitPhaseOne() routine does the first phase and should -** be invoked prior to calling this routine. The sqlite3BtreeCommitPhaseOne() -** routine did all the work of writing information out to disk and flushing the -** contents so that they are written onto the disk platter. All this -** routine has to do is delete or truncate or zero the header in the -** the rollback journal (which causes the transaction to commit) and -** drop locks. +** This routine updates the pointer map entry for page number 'key' +** so that it maps to type 'eType' and parent page number 'pgno'. ** -** This will release the write lock on the database file. If there -** are no active cursors, it also releases the read lock. +** If *pRC is initially non-zero (non-SQLITE_OK) then this routine is +** a no-op. If an error occurs, the appropriate error code is written +** into *pRC. */ -SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p){ +static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ + DbPage *pDbPage; /* The pointer map page */ + u8 *pPtrmap; /* The pointer map data */ + Pgno iPtrmap; /* The pointer map page number */ + int offset; /* Offset in pointer map page */ + int rc; /* Return code from subfunctions */ - if( p->inTrans==TRANS_NONE ) return SQLITE_OK; - sqlite3BtreeEnter(p); - btreeIntegrity(p); + if( *pRC ) return; - /* If the handle has a write-transaction open, commit the shared-btrees - ** transaction and set the shared state to TRANS_READ. - */ - if( p->inTrans==TRANS_WRITE ){ - int rc; - BtShared *pBt = p->pBt; - assert( pBt->inTransaction==TRANS_WRITE ); - assert( pBt->nTransaction>0 ); - rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); - if( rc!=SQLITE_OK ){ - sqlite3BtreeLeave(p); - return rc; + assert( sqlite3_mutex_held(pBt->mutex) ); + /* The master-journal page number must never be used as a pointer map page */ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + + assert( pBt->autoVacuum ); + if( key==0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + assert( offset <= (int)pBt->usableSize-5 ); + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ + TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + *pRC= rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + pPtrmap[offset] = eType; + put4byte(&pPtrmap[offset+1], parent); } - pBt->inTransaction = TRANS_READ; } - btreeEndTransaction(p); - sqlite3BtreeLeave(p); - return SQLITE_OK; +ptrmap_exit: + sqlite3PagerUnref(pDbPage); } /* -** Do both phases of a commit. +** Read an entry from the pointer map. +** +** This routine retrieves the pointer map entry for page 'key', writing +** the type and parent page number to *pEType and *pPgno respectively. +** An error code is returned if something goes wrong, otherwise SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ +static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ + DbPage *pDbPage; /* The pointer map page */ + int iPtrmap; /* Pointer map page index */ + u8 *pPtrmap; /* Pointer map page data */ + int offset; /* Offset of entry in pointer map */ int rc; - sqlite3BtreeEnter(p); - rc = sqlite3BtreeCommitPhaseOne(p, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeCommitPhaseTwo(p); + + assert( sqlite3_mutex_held(pBt->mutex) ); + + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); + if( rc!=0 ){ + return rc; } - sqlite3BtreeLeave(p); - return rc; -} + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); -#ifndef NDEBUG -/* -** Return the number of write-cursors open on this handle. This is for use -** in assert() expressions, so it is only compiled if NDEBUG is not -** defined. -** -** For the purposes of this routine, a write-cursor is any cursor that -** is capable of writing to the databse. That means the cursor was -** originally opened for writing and the cursor has not be disabled -** by having its state changed to CURSOR_FAULT. -*/ -static int countWriteCursors(BtShared *pBt){ - BtCursor *pCur; - int r = 0; - for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); + return SQLITE_CORRUPT_BKPT; } - return r; + assert( offset <= (int)pBt->usableSize-5 ); + assert( pEType!=0 ); + *pEType = pPtrmap[offset]; + if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); + + sqlite3PagerUnref(pDbPage); + if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_BKPT; + return SQLITE_OK; } + +#else /* if defined SQLITE_OMIT_AUTOVACUUM */ + #define ptrmapPut(w,x,y,z,rc) + #define ptrmapGet(w,x,y,z) SQLITE_OK + #define ptrmapPutOvflPtr(x, y, rc) #endif /* -** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. -** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Given a btree page and a cell index (0 means the first cell on +** the page, 1 means the second cell, and so forth) return a pointer +** to the cell content. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine works only for pages that do not contain overflow cells. */ -SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ - BtCursor *p; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; +#define findCell(P,I) \ + ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + + +/* +** This a more complex version of findCell() that works for +** pages that do contain overflow cells. +*/ +static u8 *findOverflowCell(MemPage *pPage, int iCell){ + int i; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + for(i=pPage->nOverflow-1; i>=0; i--){ + int k; + struct _OvflCell *pOvfl; + pOvfl = &pPage->aOvfl[i]; + k = pOvfl->idx; + if( k<=iCell ){ + if( k==iCell ){ + return pOvfl->pCell; + } + iCell--; } } - sqlite3BtreeLeave(pBtree); + return findCell(pPage, iCell); } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Parse a cell content block and fill in the CellInfo structure. There +** are two versions of this function. btreeParseCell() takes a +** cell index as the second argument and btreeParseCellPtr() +** takes a pointer to the body of the cell as its second argument. ** -** This will release the write lock on the database file. If there -** are no active cursors, it also releases the read lock. +** Within this file, the parseCell() macro can be called instead of +** btreeParseCellPtr(). Using some compilers, this will be faster. */ -SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p){ - int rc; - BtShared *pBt = p->pBt; - MemPage *pPage1; - - sqlite3BtreeEnter(p); - rc = saveAllCursors(pBt, 0, 0); -#ifndef SQLITE_OMIT_SHARED_CACHE - if( rc!=SQLITE_OK ){ - /* This is a horrible situation. An IO or malloc() error occurred whilst - ** trying to save cursor positions. If this is an automatic rollback (as - ** the result of a constraint, malloc() failure or IO error) then - ** the cache may be internally inconsistent (not contain valid trees) so - ** we cannot simply return the error to the caller. Instead, abort - ** all queries that may be using any of the cursors that failed to save. - */ - sqlite3BtreeTripAllCursors(p, rc); - } -#endif - btreeIntegrity(p); - - if( p->inTrans==TRANS_WRITE ){ - int rc2; +static void btreeParseCellPtr( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u16 n; /* Number bytes in cell content header */ + u32 nPayload; /* Number of bytes of cell payload */ - assert( TRANS_WRITE==pBt->inTransaction ); - rc2 = sqlite3PagerRollback(pBt->pPager); - if( rc2!=SQLITE_OK ){ - rc = rc2; - } + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - /* The rollback may have destroyed the pPage1->aData value. So - ** call btreeGetPage() on page 1 again to make - ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ - int nPage = get4byte(28+(u8*)pPage1->aData); - testcase( nPage==0 ); - if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); - testcase( pBt->nPage!=nPage ); - pBt->nPage = nPage; - releasePage(pPage1); + pInfo->pCell = pCell; + assert( pPage->leaf==0 || pPage->leaf==1 ); + n = pPage->childPtrSize; + assert( n==4-4*pPage->leaf ); + if( pPage->intKey ){ + if( pPage->hasData ){ + n += getVarint32(&pCell[n], nPayload); + }else{ + nPayload = 0; } - assert( countWriteCursors(pBt)==0 ); - pBt->inTransaction = TRANS_READ; + n += getVarint(&pCell[n], (u64*)&pInfo->nKey); + pInfo->nData = nPayload; + }else{ + pInfo->nData = 0; + n += getVarint32(&pCell[n], nPayload); + pInfo->nKey = nPayload; } + pInfo->nPayload = nPayload; + pInfo->nHeader = n; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( likely(nPayload<=pPage->maxLocal) ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + pInfo->iOverflow = 0; + }else{ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ - btreeEndTransaction(p); - sqlite3BtreeLeave(p); - return rc; -} - -/* -** Start a statement subtransaction. The subtransaction can can be rolled -** back independently of the main transaction. You must start a transaction -** before starting a subtransaction. The subtransaction is ended automatically -** if the main transaction commits or rolls back. -** -** Statement subtransactions are used around individual SQL statements -** that are contained within a BEGIN...COMMIT block. If a constraint -** error occurs within the statement, the effect of that one statement -** can be rolled back without having to rollback the entire transaction. -** -** A statement sub-transaction is implemented as an anonymous savepoint. The -** value passed as the second parameter is the total number of savepoints, -** including the new anonymous savepoint, open on the B-Tree. i.e. if there -** are no active savepoints and no other statement-transactions open, -** iStatement is 1. This anonymous savepoint can be released or rolled back -** using the sqlite3BtreeSavepoint() function. -*/ -SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ - int rc; - BtShared *pBt = p->pBt; - sqlite3BtreeEnter(p); - assert( p->inTrans==TRANS_WRITE ); - assert( pBt->readOnly==0 ); - assert( iStatement>0 ); - assert( iStatement>p->db->nSavepoint ); - assert( pBt->inTransaction==TRANS_WRITE ); - /* At the pager level, a statement transaction is a savepoint with - ** an index greater than all savepoints created explicitly using - ** SQL statements. It is illegal to open, release or rollback any - ** such savepoints while the statement transaction savepoint is active. - */ - rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); - sqlite3BtreeLeave(p); - return rc; -} - -/* -** The second argument to this function, op, is always SAVEPOINT_ROLLBACK -** or SAVEPOINT_RELEASE. This function either releases or rolls back the -** savepoint identified by parameter iSavepoint, depending on the value -** of op. -** -** Normally, iSavepoint is greater than or equal to zero. However, if op is -** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the -** contents of the entire transaction are rolled back. This is different -** from a normal transaction rollback, as no locks are released and the -** transaction remains open. -*/ -SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ - int rc = SQLITE_OK; - if( p && p->inTrans==TRANS_WRITE ){ - BtShared *pBt = p->pBt; - assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); - assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); - sqlite3BtreeEnter(p); - rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); - if( rc==SQLITE_OK ){ - if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; - rc = newDatabase(pBt); - pBt->nPage = get4byte(28 + pBt->pPage1->aData); - - /* The database size was written into the offset 28 of the header - ** when the transaction started, so we know that the value at offset - ** 28 is nonzero. */ - assert( pBt->nPage>0 ); + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; } - sqlite3BtreeLeave(p); + pInfo->iOverflow = (u16)(pInfo->nLocal + n); + pInfo->nSize = pInfo->iOverflow + 4; } - return rc; +} +#define parseCell(pPage, iCell, pInfo) \ + btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) +static void btreeParseCell( + MemPage *pPage, /* Page containing the cell */ + int iCell, /* The cell index. First cell is 0 */ + CellInfo *pInfo /* Fill in this structure */ +){ + parseCell(pPage, iCell, pInfo); } /* -** Create a new cursor for the BTree whose root is on the page -** iTable. If a read-only cursor is requested, it is assumed that -** the caller already has at least a read-only transaction open -** on the database already. If a write-cursor is requested, then -** the caller is assumed to have an open write transaction. -** -** If wrFlag==0, then the cursor can only be used for reading. -** If wrFlag==1, then the cursor can be used for reading or for -** writing if other conditions for writing are also met. These -** are the conditions that must be met in order for writing to -** be allowed: -** -** 1: The cursor must have been opened with wrFlag==1 -** -** 2: Other database connections that share the same pager cache -** but which are not in the READ_UNCOMMITTED state may not have -** cursors open with wrFlag==0 on the same table. Otherwise -** the changes made by this write cursor would be visible to -** the read cursors in the other database connection. -** -** 3: The database must be writable (not on read-only media) -** -** 4: There must be an active transaction. -** -** No checking is done to make sure that page iTable really is the -** root page of a b-tree. If it is not, then the cursor acquired -** will not work correctly. -** -** It is assumed that the sqlite3BtreeCursorZero() has been called -** on pCur to initialize the memory space prior to invoking this routine. +** Compute the total number of bytes that a Cell needs in the cell +** data area of the btree-page. The return number includes the cell +** data header and the local payload, but not any overflow page or +** the space used by the cell pointer. */ -static int btreeCursor( - Btree *p, /* The btree */ - int iTable, /* Root page of table to open */ - int wrFlag, /* 1 to write. 0 read-only */ - struct KeyInfo *pKeyInfo, /* First arg to comparison function */ - BtCursor *pCur /* Space for new cursor */ -){ - BtShared *pBt = p->pBt; /* Shared b-tree handle */ - - assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 || wrFlag==1 ); +static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ + u8 *pIter = &pCell[pPage->childPtrSize]; + u32 nSize; - /* The following assert statements verify that if this is a sharable - ** b-tree database, the connection is holding the required table locks, - ** and that no other connection has any open cursor that conflicts with - ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); - assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + btreeParseCellPtr(pPage, pCell, &debuginfo); +#endif - /* Assert that the caller has opened the required transaction. */ - assert( p->inTrans>TRANS_NONE ); - assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); - assert( pBt->pPage1 && pBt->pPage1->aData ); + if( pPage->intKey ){ + u8 *pEnd; + if( pPage->hasData ){ + pIter += getVarint32(pIter, nSize); + }else{ + nSize = 0; + } - if( NEVER(wrFlag && pBt->readOnly) ){ - return SQLITE_READONLY; + /* pIter now points at the 64-bit integer key value, a variable length + ** integer. The following block moves pIter to point at the first byte + ** past the end of the key value. */ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pItermaxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4; } + nSize += (u32)(pIter - pCell); - /* Now that no other errors can occur, finish filling in the BtCursor - ** variables and link the cursor into the BtShared list. */ - pCur->pgnoRoot = (Pgno)iTable; - pCur->iPage = -1; - pCur->pKeyInfo = pKeyInfo; - pCur->pBtree = p; - pCur->pBt = pBt; - pCur->wrFlag = (u8)wrFlag; - pCur->pNext = pBt->pCursor; - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur; + /* The minimum size of any cell is 4 bytes. */ + if( nSize<4 ){ + nSize = 4; } - pBt->pCursor = pCur; - pCur->eState = CURSOR_INVALID; - pCur->cachedRowid = 0; - return SQLITE_OK; -} -SQLITE_PRIVATE int sqlite3BtreeCursor( - Btree *p, /* The btree */ - int iTable, /* Root page of table to open */ - int wrFlag, /* 1 to write. 0 read-only */ - struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ - BtCursor *pCur /* Write new cursor here */ -){ - int rc; - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); - return rc; -} -/* -** Return the size of a BtCursor object in bytes. -** -** This interfaces is needed so that users of cursors can preallocate -** sufficient storage to hold a cursor. The BtCursor object is opaque -** to users so they cannot do the sizeof() themselves - they must call -** this routine. -*/ -SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){ - return ROUND8(sizeof(BtCursor)); + assert( nSize==debuginfo.nSize ); + return (u16)nSize; } -/* -** Initialize memory that will be converted into a BtCursor object. -** -** The simple approach here would be to memset() the entire object -** to zero. But it turns out that the apPage[] and aiIdx[] arrays -** do not need to be zeroed and they are large, so we can save a lot -** of run-time by skipping the initialization of those elements. -*/ -SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ - memset(p, 0, offsetof(BtCursor, iPage)); +#ifdef SQLITE_DEBUG +/* This variation on cellSizePtr() is used inside of assert() statements +** only. */ +static u16 cellSize(MemPage *pPage, int iCell){ + return cellSizePtr(pPage, findCell(pPage, iCell)); } +#endif +#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Set the cached rowid value of every cursor in the same database file -** as pCur and having the same root page number as pCur. The value is -** set to iRowid. -** -** Only positive rowid values are considered valid for this cache. -** The cache is initialized to zero, indicating an invalid cache. -** A btree will work fine with zero or negative rowids. We just cannot -** cache zero or negative rowids, which means tables that use zero or -** negative rowids might run a little slower. But in practice, zero -** or negative rowids are very uncommon so this should not be a problem. +** If the cell pCell, part of page pPage contains a pointer +** to an overflow page, insert an entry into the pointer-map +** for the overflow page. */ -SQLITE_PRIVATE void sqlite3BtreeSetCachedRowid(BtCursor *pCur, sqlite3_int64 iRowid){ - BtCursor *p; - for(p=pCur->pBt->pCursor; p; p=p->pNext){ - if( p->pgnoRoot==pCur->pgnoRoot ) p->cachedRowid = iRowid; +static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ + CellInfo info; + if( *pRC ) return; + assert( pCell!=0 ); + btreeParseCellPtr(pPage, pCell, &info); + assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); + if( info.iOverflow ){ + Pgno ovfl = get4byte(&pCell[info.iOverflow]); + ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } - assert( pCur->cachedRowid==iRowid ); } +#endif -/* -** Return the cached rowid for the given cursor. A negative or zero -** return value indicates that the rowid cache is invalid and should be -** ignored. If the rowid cache has never before been set, then a -** zero is returned. -*/ -SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeGetCachedRowid(BtCursor *pCur){ - return pCur->cachedRowid; -} /* -** Close a cursor. The read lock on the database file is released -** when the last cursor is closed. +** Defragment the page given. All Cells are moved to the +** end of the page and all free space is collected into one +** big FreeBlk that occurs in between the header and cell +** pointer array and the cell content area. */ -SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ - Btree *pBtree = pCur->pBtree; - if( pBtree ){ - int i; - BtShared *pBt = pCur->pBt; - sqlite3BtreeEnter(pBtree); - sqlite3BtreeClearCursor(pCur); - if( pCur->pPrev ){ - pCur->pPrev->pNext = pCur->pNext; - }else{ - pBt->pCursor = pCur->pNext; +static int defragmentPage(MemPage *pPage){ + int i; /* Loop counter */ + int pc; /* Address of a i-th cell */ + int hdr; /* Offset to the page header */ + int size; /* Size of a cell */ + int usableSize; /* Number of usable bytes on a page */ + int cellOffset; /* Offset to the cell pointer array */ + int cbrk; /* Offset to the cell content area */ + int nCell; /* Number of cells on the page */ + unsigned char *data; /* The page data */ + unsigned char *temp; /* Temp area for cell content */ + int iCellFirst; /* First allowable cell index */ + int iCellLast; /* Last possible cell index */ + + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt!=0 ); + assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); + assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + data = pPage->aData; + hdr = pPage->hdrOffset; + cellOffset = pPage->cellOffset; + nCell = pPage->nCell; + assert( nCell==get2byte(&data[hdr+3]) ); + usableSize = pPage->pBt->usableSize; + cbrk = get2byte(&data[hdr+5]); + memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); + cbrk = usableSize; + iCellFirst = cellOffset + 2*nCell; + iCellLast = usableSize - 4; + for(i=0; iiCellLast ){ + return SQLITE_CORRUPT_BKPT; } - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur->pPrev; +#endif + assert( pc>=iCellFirst && pc<=iCellLast ); + size = cellSizePtr(pPage, &temp[pc]); + cbrk -= size; +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + if( cbrkiPage; i++){ - releasePage(pCur->apPage[i]); +#else + if( cbrkusableSize ){ + return SQLITE_CORRUPT_BKPT; } - unlockBtreeIfUnused(pBt); - invalidateOverflowCache(pCur); - /* sqlite3_free(pCur); */ - sqlite3BtreeLeave(pBtree); +#endif + assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + memcpy(&data[cbrk], &temp[pc], size); + put2byte(pAddr, cbrk); + } + assert( cbrk>=iCellFirst ); + put2byte(&data[hdr+5], cbrk); + data[hdr+1] = 0; + data[hdr+2] = 0; + data[hdr+7] = 0; + memset(&data[iCellFirst], 0, cbrk-iCellFirst); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_BKPT; } return SQLITE_OK; } /* -** Make sure the BtCursor* given in the argument has a valid -** BtCursor.info structure. If it is not already valid, call -** btreeParseCell() to fill it in. -** -** BtCursor.info is a cache of the information in the current cell. -** Using this cache reduces the number of calls to btreeParseCell(). +** Allocate nByte bytes of space from within the B-Tree page passed +** as the first argument. Write into *pIdx the index into pPage->aData[] +** of the first byte of allocated space. Return either SQLITE_OK or +** an error code (usually SQLITE_CORRUPT). ** -** 2007-06-25: There is a bug in some versions of MSVC that cause the -** compiler to crash when getCellInfo() is implemented as a macro. -** But there is a measureable speed advantage to using the macro on gcc -** (when less compiler optimizations like -Os or -O0 are used and the -** compiler is not doing agressive inlining.) So we use a real function -** for MSVC and a macro for everything else. Ticket #2457. +** The caller guarantees that there is sufficient space to make the +** allocation. This routine might need to defragment in order to bring +** all the space together, however. This routine will avoid using +** the first two bytes past the cell pointer area since presumably this +** allocation is being made in order to insert a new cell, so we will +** also end up needing a new cell pointer. */ -#ifndef NDEBUG - static void assertCellInfo(BtCursor *pCur){ - CellInfo info; - int iPage = pCur->iPage; - memset(&info, 0, sizeof(info)); - btreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); - assert( memcmp(&info, &pCur->info, sizeof(info))==0 ); - } -#else - #define assertCellInfo(x) -#endif -#ifdef _MSC_VER - /* Use a real function in MSVC to work around bugs in that compiler. */ - static void getCellInfo(BtCursor *pCur){ - if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - pCur->validNKey = 1; - }else{ - assertCellInfo(pCur); +static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ + const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ + u8 * const data = pPage->aData; /* Local cache of pPage->aData */ + int nFrag; /* Number of fragmented bytes on pPage */ + int top; /* First byte of cell content area */ + int gap; /* First byte of gap between cell pointers and cell content */ + int rc; /* Integer return code */ + int usableSize; /* Usable size of the page */ + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nByte>=0 ); /* Minimum cell size is 4 */ + assert( pPage->nFree>=nByte ); + assert( pPage->nOverflow==0 ); + usableSize = pPage->pBt->usableSize; + assert( nByte < usableSize-8 ); + + nFrag = data[hdr+7]; + assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); + gap = pPage->cellOffset + 2*pPage->nCell; + top = get2byteNotZero(&data[hdr+5]); + if( gap>top ) return SQLITE_CORRUPT_BKPT; + testcase( gap+2==top ); + testcase( gap+1==top ); + testcase( gap==top ); + + if( nFrag>=60 ){ + /* Always defragment highly fragmented pages */ + rc = defragmentPage(pPage); + if( rc ) return rc; + top = get2byteNotZero(&data[hdr+5]); + }else if( gap+2<=top ){ + /* Search the freelist looking for a free slot big enough to satisfy + ** the request. The allocation is made from the first free slot in + ** the list that is large enough to accomadate it. + */ + int pc, addr; + for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&data[addr], &data[pc], 2); + data[hdr+7] = (u8)(nFrag + x); + }else if( size+pc > usableSize ){ + return SQLITE_CORRUPT_BKPT; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&data[pc+2], x); + } + *pIdx = pc + x; + return SQLITE_OK; + } } } -#else /* if not _MSC_VER */ - /* Use a macro in all other compilers so that the function is inlined */ -#define getCellInfo(pCur) \ - if( pCur->info.nSize==0 ){ \ - int iPage = pCur->iPage; \ - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ - pCur->validNKey = 1; \ - }else{ \ - assertCellInfo(pCur); \ + + /* Check to make sure there is enough space in the gap to satisfy + ** the allocation. If not, defragment. + */ + testcase( gap+2+nByte==top ); + if( gap+2+nByte>top ){ + rc = defragmentPage(pPage); + if( rc ) return rc; + top = get2byteNotZero(&data[hdr+5]); + assert( gap+nByte<=top ); } -#endif /* _MSC_VER */ -#ifndef NDEBUG /* The next routine used only within assert() statements */ -/* -** Return true if the given BtCursor is valid. A valid cursor is one -** that is currently pointing to a row in a (non-empty) table. -** This is a verification routine is used only within assert() statements. -*/ -SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ - return pCur && pCur->eState==CURSOR_VALID; -} -#endif /* NDEBUG */ -/* -** Set *pSize to the size of the buffer needed to hold the value of -** the key for the current entry. If the cursor is not pointing -** to a valid entry, *pSize is set to 0. -** -** For a table with the INTKEY flag set, this routine returns the key -** itself, not the number of bytes in the key. -** -** The caller must position the cursor prior to invoking this routine. -** -** This routine cannot fail. It always returns SQLITE_OK. -*/ -SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); - if( pCur->eState!=CURSOR_VALID ){ - *pSize = 0; - }else{ - getCellInfo(pCur); - *pSize = pCur->info.nKey; - } + /* Allocate memory from the gap in between the cell pointer array + ** and the cell content area. The btreeInitPage() call has already + ** validated the freelist. Given that the freelist is valid, there + ** is no way that the allocation can extend off the end of the page. + ** The assert() below verifies the previous sentence. + */ + top -= nByte; + put2byte(&data[hdr+5], top); + assert( top+nByte <= (int)pPage->pBt->usableSize ); + *pIdx = top; return SQLITE_OK; } /* -** Set *pSize to the number of bytes of data in the entry the -** cursor currently points to. -** -** The caller must guarantee that the cursor is pointing to a non-NULL -** valid entry. In other words, the calling procedure must guarantee -** that the cursor has Cursor.eState==CURSOR_VALID. +** Return a section of the pPage->aData to the freelist. +** The first byte of the new free block is pPage->aDisk[start] +** and the size of the block is "size" bytes. ** -** Failure is not possible. This function always returns SQLITE_OK. -** It might just as well be a procedure (returning void) but we continue -** to return an integer result code for historical reasons. +** Most of the effort here is involved in coalesing adjacent +** free blocks into a single big free block. */ -SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - getCellInfo(pCur); - *pSize = pCur->info.nData; - return SQLITE_OK; -} +static int freeSpace(MemPage *pPage, int start, int size){ + int addr, pbegin, hdr; + int iLast; /* Largest possible freeblock offset */ + unsigned char *data = pPage->aData; -/* -** Given the page number of an overflow page in the database (parameter -** ovfl), this function finds the page number of the next page in the -** linked list of overflow pages. If possible, it uses the auto-vacuum -** pointer-map data instead of reading the content of page ovfl to do so. -** -** If an error occurs an SQLite error code is returned. Otherwise: -** -** The page number of the next overflow page in the linked list is -** written to *pPgnoNext. If page ovfl is the last page in its linked -** list, *pPgnoNext is set to zero. -** -** If ppPage is not NULL, and a reference to the MemPage object corresponding -** to page number pOvfl was obtained, then *ppPage is set to point to that -** reference. It is the responsibility of the caller to call releasePage() -** on *ppPage to free the reference. In no reference was obtained (because -** the pointer-map was used to obtain the value for *pPgnoNext), then -** *ppPage is set to zero. -*/ -static int getOverflowPage( - BtShared *pBt, /* The database file */ - Pgno ovfl, /* Current overflow page number */ - MemPage **ppPage, /* OUT: MemPage handle (may be NULL) */ - Pgno *pPgnoNext /* OUT: Next overflow page number */ -){ - Pgno next = 0; - MemPage *pPage = 0; - int rc = SQLITE_OK; + assert( pPage->pBt!=0 ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( (start + size) <= (int)pPage->pBt->usableSize ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( size>=0 ); /* Minimum cell size is 4 */ - assert( sqlite3_mutex_held(pBt->mutex) ); - assert(pPgnoNext); + if( pPage->pBt->secureDelete ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[start], 0, size); + } -#ifndef SQLITE_OMIT_AUTOVACUUM - /* Try to find the next page in the overflow list using the - ** autovacuum pointer-map pages. Guess that the next page in - ** the overflow list is page number (ovfl+1). If that guess turns - ** out to be wrong, fall back to loading the data of page - ** number ovfl to determine the next page number. + /* Add the space back into the linked list of freeblocks. Note that + ** even though the freeblock list was checked by btreeInitPage(), + ** btreeInitPage() did not detect overlapping cells or + ** freeblocks that overlapped cells. Nor does it detect when the + ** cell content area exceeds the value in the page header. If these + ** situations arise, then subsequent insert operations might corrupt + ** the freelist. So we do need to check for corruption while scanning + ** the freelist. */ - if( pBt->autoVacuum ){ - Pgno pgno; - Pgno iGuess = ovfl+1; - u8 eType; - - while( PTRMAP_ISPAGE(pBt, iGuess) || iGuess==PENDING_BYTE_PAGE(pBt) ){ - iGuess++; - } - - if( iGuess<=btreePagecount(pBt) ){ - rc = ptrmapGet(pBt, iGuess, &eType, &pgno); - if( rc==SQLITE_OK && eType==PTRMAP_OVERFLOW2 && pgno==ovfl ){ - next = iGuess; - rc = SQLITE_DONE; - } + hdr = pPage->hdrOffset; + addr = hdr + 1; + iLast = pPage->pBt->usableSize - 4; + assert( start<=iLast ); + while( (pbegin = get2byte(&data[addr]))0 ){ + if( pbeginiLast ){ + return SQLITE_CORRUPT_BKPT; + } + assert( pbegin>addr || pbegin==0 ); + put2byte(&data[addr], start); + put2byte(&data[start], pbegin); + put2byte(&data[start+2], size); + pPage->nFree = pPage->nFree + (u16)size; - assert( next==0 || rc==SQLITE_DONE ); - if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); - assert( rc==SQLITE_OK || pPage==0 ); - if( rc==SQLITE_OK ){ - next = get4byte(pPage->aData); + /* Coalesce adjacent free blocks */ + addr = hdr + 1; + while( (pbegin = get2byte(&data[addr]))>0 ){ + int pnext, psize, x; + assert( pbegin>addr ); + assert( pbegin <= (int)pPage->pBt->usableSize-4 ); + pnext = get2byte(&data[pbegin]); + psize = get2byte(&data[pbegin+2]); + if( pbegin + psize + 3 >= pnext && pnext>0 ){ + int frag = pnext - (pbegin+psize); + if( (frag<0) || (frag>(int)data[hdr+7]) ){ + return SQLITE_CORRUPT_BKPT; + } + data[hdr+7] -= (u8)frag; + x = get2byte(&data[pnext]); + put2byte(&data[pbegin], x); + x = pnext + get2byte(&data[pnext+2]) - pbegin; + put2byte(&data[pbegin+2], x); + }else{ + addr = pbegin; } } - *pPgnoNext = next; - if( ppPage ){ - *ppPage = pPage; - }else{ - releasePage(pPage); + /* If the cell content area begins with a freeblock, remove it. */ + if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ + int top; + pbegin = get2byte(&data[hdr+1]); + memcpy(&data[hdr+1], &data[pbegin], 2); + top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); + put2byte(&data[hdr+5], top); } - return (rc==SQLITE_DONE ? SQLITE_OK : rc); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + return SQLITE_OK; } /* -** Copy data from a buffer to a page, or from a page to a buffer. +** Decode the flags byte (the first byte of the header) for a page +** and initialize fields of the MemPage structure accordingly. ** -** pPayload is a pointer to data stored on database page pDbPage. -** If argument eOp is false, then nByte bytes of data are copied -** from pPayload to the buffer pointed at by pBuf. If eOp is true, -** then sqlite3PagerWrite() is called on pDbPage and nByte bytes -** of data are copied from the buffer pBuf to pPayload. +** Only the following combinations are supported. Anything different +** indicates a corrupt database files: ** -** SQLITE_OK is returned on success, otherwise an error code. +** PTF_ZERODATA +** PTF_ZERODATA | PTF_LEAF +** PTF_LEAFDATA | PTF_INTKEY +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF */ -static int copyPayload( - void *pPayload, /* Pointer to page data */ - void *pBuf, /* Pointer to buffer */ - int nByte, /* Number of bytes to copy */ - int eOp, /* 0 -> copy from page, 1 -> copy to page */ - DbPage *pDbPage /* Page containing pPayload */ -){ - if( eOp ){ - /* Copy data from buffer to page (a write operation) */ - int rc = sqlite3PagerWrite(pDbPage); - if( rc!=SQLITE_OK ){ - return rc; - } - memcpy(pPayload, pBuf, nByte); +static int decodeFlags(MemPage *pPage, int flagByte){ + BtShared *pBt; /* A copy of pPage->pBt */ + + assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); + flagByte &= ~PTF_LEAF; + pPage->childPtrSize = 4-4*pPage->leaf; + pBt = pPage->pBt; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + pPage->intKey = 1; + pPage->hasData = pPage->leaf; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==PTF_ZERODATA ){ + pPage->intKey = 0; + pPage->hasData = 0; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; }else{ - /* Copy data from page to buffer (a read operation) */ - memcpy(pBuf, pPayload, nByte); + return SQLITE_CORRUPT_BKPT; } return SQLITE_OK; } /* -** This function is used to read or overwrite payload information -** for the entry that the pCur cursor is pointing to. If the eOp -** parameter is 0, this is a read operation (data copied into -** buffer pBuf). If it is non-zero, a write (data copied from -** buffer pBuf). -** -** A total of "amt" bytes are read or written beginning at "offset". -** Data is read to or from the buffer pBuf. -** -** The content being read or written might appear on the main page -** or be scattered out on multiple overflow pages. -** -** If the BtCursor.isIncrblobHandle flag is set, and the current -** cursor entry uses one or more overflow pages, this function -** allocates space for and lazily popluates the overflow page-list -** cache array (BtCursor.aOverflow). Subsequent calls use this -** cache to make seeking to the supplied offset more efficient. -** -** Once an overflow page-list cache has been allocated, it may be -** invalidated if some other cursor writes to the same table, or if -** the cursor is moved to a different row. Additionally, in auto-vacuum -** mode, the following events may invalidate an overflow page-list cache. +** Initialize the auxiliary information for a disk block. ** -** * An incremental vacuum, -** * A commit in auto_vacuum="full" mode, -** * Creating a table (may require moving an overflow page). +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. */ -static int accessPayload( - BtCursor *pCur, /* Cursor pointing to entry to read from */ - u32 offset, /* Begin reading this far into payload */ - u32 amt, /* Read this many bytes */ - unsigned char *pBuf, /* Write the bytes into this buffer */ - int eOp /* zero to read. non-zero to write. */ -){ - unsigned char *aPayload; - int rc = SQLITE_OK; - u32 nKey; - int iIdx = 0; - MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ - BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ - - assert( pPage ); - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->aiIdx[pCur->iPage]nCell ); - assert( cursorHoldsMutex(pCur) ); - - getCellInfo(pCur); - aPayload = pCur->info.pCell + pCur->info.nHeader; - nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); - - if( NEVER(offset+amt > nKey+pCur->info.nData) - || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] - ){ - /* Trying to read or write past the end of the data is an error */ - return SQLITE_CORRUPT_BKPT; - } +static int btreeInitPage(MemPage *pPage){ - /* Check if data must be read/written to/from the btree page itself. */ - if( offsetinfo.nLocal ){ - int a = amt; - if( a+offset>pCur->info.nLocal ){ - a = pCur->info.nLocal - offset; - } - rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); - offset = 0; - pBuf += a; - amt -= a; - }else{ - offset -= pCur->info.nLocal; - } + assert( pPage->pBt!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); - if( rc==SQLITE_OK && amt>0 ){ - const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ - Pgno nextPage; + if( !pPage->isInit ){ + u16 pc; /* Address of a freeblock within pPage->aData[] */ + u8 hdr; /* Offset to beginning of page header */ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + int usableSize; /* Amount of usable space on each page */ + u16 cellOffset; /* Offset from start of page to first cell pointer */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ - nextPage = get4byte(&aPayload[pCur->info.nLocal]); + pBt = pPage->pBt; -#ifndef SQLITE_OMIT_INCRBLOB - /* If the isIncrblobHandle flag is set and the BtCursor.aOverflow[] - ** has not been allocated, allocate it now. The array is sized at - ** one entry for each overflow page in the overflow chain. The - ** page number of the first overflow page is stored in aOverflow[0], - ** etc. A value of 0 in the aOverflow[] array means "not yet known" - ** (the cache is lazily populated). - */ - if( pCur->isIncrblobHandle && !pCur->aOverflow ){ - int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - pCur->aOverflow = (Pgno *)sqlite3MallocZero(sizeof(Pgno)*nOvfl); - /* nOvfl is always positive. If it were zero, fetchPayload would have - ** been used instead of this routine. */ - if( ALWAYS(nOvfl) && !pCur->aOverflow ){ - rc = SQLITE_NOMEM; - } + hdr = pPage->hdrOffset; + data = pPage->aData; + if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + usableSize = pBt->usableSize; + pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + top = get2byteNotZero(&data[hdr+5]); + pPage->nCell = get2byte(&data[hdr+3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_BKPT; } + testcase( pPage->nCell==MX_CELL(pBt) ); - /* If the overflow page-list cache has been allocated and the - ** entry for the first required overflow page is valid, skip - ** directly to it. + /* A malformed database page might cause us to read past the end + ** of page when parsing a cell. + ** + ** The following block of code checks early to see if a cell extends + ** past the end of a page boundary and causes SQLITE_CORRUPT to be + ** returned if it does. */ - if( pCur->aOverflow && pCur->aOverflow[offset/ovflSize] ){ - iIdx = (offset/ovflSize); - nextPage = pCur->aOverflow[iIdx]; - offset = (offset%ovflSize); - } -#endif - - for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ + iCellFirst = cellOffset + 2*pPage->nCell; + iCellLast = usableSize - 4; +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + { + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ -#ifndef SQLITE_OMIT_INCRBLOB - /* If required, populate the overflow page-list cache. */ - if( pCur->aOverflow ){ - assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); - pCur->aOverflow[iIdx] = nextPage; + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byte(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_BKPT; + } + sz = cellSizePtr(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_BKPT; + } } + if( !pPage->leaf ) iCellLast++; + } #endif - if( offset>=ovflSize ){ - /* The only reason to read this page is to obtain the page - ** number for the next page in the overflow chain. The page - ** data is not required. So first try to lookup the overflow - ** page-list cache, if any, then fall back to the getOverflowPage() - ** function. - */ -#ifndef SQLITE_OMIT_INCRBLOB - if( pCur->aOverflow && pCur->aOverflow[iIdx+1] ){ - nextPage = pCur->aOverflow[iIdx+1]; - } else -#endif - rc = getOverflowPage(pBt, nextPage, 0, &nextPage); - offset -= ovflSize; - }else{ - /* Need to read this page properly. It contains some of the - ** range of data that is being read (eOp==0) or written (eOp!=0). - */ - DbPage *pDbPage; - int a = amt; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); - if( rc==SQLITE_OK ){ - aPayload = sqlite3PagerGetData(pDbPage); - nextPage = get4byte(aPayload); - if( a + offset > ovflSize ){ - a = ovflSize - offset; - } - rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); - sqlite3PagerUnref(pDbPage); - offset = 0; - amt -= a; - pBuf += a; - } + /* Compute the total free space on the page */ + pc = get2byte(&data[hdr+1]); + nFree = data[hdr+7] + top; + while( pc>0 ){ + u16 next, size; + if( pciCellLast ){ + /* Start of free block is off the page */ + return SQLITE_CORRUPT_BKPT; + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); + if( (next>0 && next<=pc+size+3) || pc+size>usableSize ){ + /* Free blocks must be in ascending order. And the last byte of + ** the free-block must lie on the database page. */ + return SQLITE_CORRUPT_BKPT; } + nFree = nFree + size; + pc = next; } - } - if( rc==SQLITE_OK && amt>0 ){ - return SQLITE_CORRUPT_BKPT; + /* At this point, nFree contains the sum of the offset to the start + ** of the cell-content area plus the number of free bytes within + ** the cell-content area. If this is greater than the usable-size + ** of the page, then the page must be corrupted. This check also + ** serves to verify that the offset to the start of the cell-content + ** area, according to the page header, lies within the page. + */ + if( nFree>usableSize ){ + return SQLITE_CORRUPT_BKPT; + } + pPage->nFree = (u16)(nFree - iCellFirst); + pPage->isInit = 1; } - return rc; -} - -/* -** Read part of the key associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer -** begins at "offset". -** -** The caller must ensure that pCur is pointing to a valid row -** in the table. -** -** Return SQLITE_OK on success or an error code if anything goes -** wrong. An error is returned if "offset+amt" is larger than -** the available payload. -*/ -SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); + return SQLITE_OK; } /* -** Read part of the data associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer -** begins at "offset". -** -** Return SQLITE_OK on success or an error code if anything goes -** wrong. An error is returned if "offset+amt" is larger than -** the available payload. +** Set up a raw page so that it looks like a database page holding +** no entries. */ -SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ - int rc; +static void zeroPage(MemPage *pPage, int flags){ + unsigned char *data = pPage->aData; + BtShared *pBt = pPage->pBt; + u8 hdr = pPage->hdrOffset; + u16 first; -#ifndef SQLITE_OMIT_INCRBLOB - if ( pCur->eState==CURSOR_INVALID ){ - return SQLITE_ABORT; - } -#endif - - assert( cursorHoldsMutex(pCur) ); - rc = restoreCursorPosition(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - rc = accessPayload(pCur, offset, amt, pBuf, 0); - } - return rc; -} - -/* -** Return a pointer to payload information from the entry that the -** pCur cursor is pointing to. The pointer is to the beginning of -** the key if skipKey==0 and it points to the beginning of data if -** skipKey==1. The number of bytes of available key/data is written -** into *pAmt. If *pAmt==0, then the value returned will not be -** a valid pointer. -** -** This routine is an optimization. It is common for the entire key -** and data to fit on the local page and for there to be no overflow -** pages. When that is so, this routine can be used to access the -** key and data without making a copy. If the key and/or data spills -** onto overflow pages, then accessPayload() must be used to reassemble -** the key/data and copy it into a preallocated buffer. -** -** The pointer returned by this routine looks directly into the cached -** page of the database. The data might change or move the next time -** any btree routine is called. -*/ -static const unsigned char *fetchPayload( - BtCursor *pCur, /* Cursor pointing to entry to read from */ - int *pAmt, /* Write the number of available bytes here */ - int skipKey /* read beginning at data if this is true */ -){ - unsigned char *aPayload; - MemPage *pPage; - u32 nKey; - u32 nLocal; - - assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); - assert( pCur->eState==CURSOR_VALID ); - assert( cursorHoldsMutex(pCur) ); - pPage = pCur->apPage[pCur->iPage]; - assert( pCur->aiIdx[pCur->iPage]nCell ); - if( NEVER(pCur->info.nSize==0) ){ - btreeParseCell(pCur->apPage[pCur->iPage], pCur->aiIdx[pCur->iPage], - &pCur->info); - } - aPayload = pCur->info.pCell; - aPayload += pCur->info.nHeader; - if( pPage->intKey ){ - nKey = 0; - }else{ - nKey = (int)pCur->info.nKey; - } - if( skipKey ){ - aPayload += nKey; - nLocal = pCur->info.nLocal - nKey; - }else{ - nLocal = pCur->info.nLocal; - assert( nLocal<=nKey ); + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage) == data ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->secureDelete ){ + memset(&data[hdr], 0, pBt->usableSize - hdr); } - *pAmt = nLocal; - return aPayload; + data[hdr] = (char)flags; + first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pBt->usableSize); + pPage->nFree = (u16)(pBt->usableSize - first); + decodeFlags(pPage, flags); + pPage->hdrOffset = hdr; + pPage->cellOffset = first; + pPage->nOverflow = 0; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nCell = 0; + pPage->isInit = 1; } /* -** For the entry that cursor pCur is point to, return as -** many bytes of the key or data as are available on the local -** b-tree page. Write the number of available bytes into *pAmt. -** -** The pointer returned is ephemeral. The key/data may move -** or be destroyed on the next call to any Btree routine, -** including calls from other threads against the same cache. -** Hence, a mutex on the BtShared should be held prior to calling -** this routine. -** -** These routines is used to get quick access to key and data -** in the common case where no overflow pages are used. +** Convert a DbPage obtained from the pager into a MemPage used by +** the btree layer. */ -SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor *pCur, int *pAmt){ - const void *p = 0; - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert( cursorHoldsMutex(pCur) ); - if( ALWAYS(pCur->eState==CURSOR_VALID) ){ - p = (const void*)fetchPayload(pCur, pAmt, 0); - } - return p; -} -SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, int *pAmt){ - const void *p = 0; - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert( cursorHoldsMutex(pCur) ); - if( ALWAYS(pCur->eState==CURSOR_VALID) ){ - p = (const void*)fetchPayload(pCur, pAmt, 1); - } - return p; +static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ + MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; + return pPage; } - /* -** Move the cursor down to a new child page. The newPgno argument is the -** page number of the child page to move to. +** Get a page from the pager. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. ** -** This function returns SQLITE_CORRUPT if the page-header flags field of -** the new child page does not match the flags field of the parent (i.e. -** if an intkey page appears to be the parent of a non-intkey page, or -** vice-versa). +** If the noContent flag is set, it means that we do not care about +** the content of the page at this time. So do not go to the disk +** to fetch the content. Just fill in the content with zeros for now. +** If in the future we call sqlite3PagerWrite() on this page, that +** means we have started to be concerned about content and the disk +** read should occur at that point. */ -static int moveToChild(BtCursor *pCur, u32 newPgno){ +static int btreeGetPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int noContent /* Do not load page content if true */ +){ int rc; - int i = pCur->iPage; - MemPage *pNewPage; - BtShared *pBt = pCur->pBt; + DbPage *pDbPage; - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPageiPage>=(BTCURSOR_MAX_DEPTH-1) ){ - return SQLITE_CORRUPT_BKPT; - } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + assert( sqlite3_mutex_held(pBt->mutex) ); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); if( rc ) return rc; - pCur->apPage[i+1] = pNewPage; - pCur->aiIdx[i+1] = 0; - pCur->iPage++; - - pCur->info.nSize = 0; - pCur->validNKey = 0; - if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ - return SQLITE_CORRUPT_BKPT; - } + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; } -#ifndef NDEBUG /* -** Page pParent is an internal (non-leaf) tree page. This function -** asserts that page number iChild is the left-child if the iIdx'th -** cell in page pParent. Or, if iIdx is equal to the total number of -** cells in pParent, that page number iChild is the right-child of -** the page. +** Retrieve a page from the pager cache. If the requested page is not +** already in the pager cache return NULL. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. */ -static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ - assert( iIdx<=pParent->nCell ); - if( iIdx==pParent->nCell ){ - assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); - }else{ - assert( get4byte(findCell(pParent, iIdx))==iChild ); +static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ + DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); + pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); + if( pDbPage ){ + return btreePageFromDbPage(pDbPage, pgno, pBt); } + return 0; } -#else -# define assertParentIndex(x,y,z) -#endif /* -** Move the cursor up to the parent page. -** -** pCur->idx is set to the cell index that contains the pointer -** to the page we are coming from. If we are coming from the -** right-most child page then pCur->idx is set to one more than -** the largest cell index. +** Return the size of the database file in pages. If there is any kind of +** error, return ((unsigned int)-1). */ -static void moveToParent(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>0 ); - assert( pCur->apPage[pCur->iPage] ); - assertParentIndex( - pCur->apPage[pCur->iPage-1], - pCur->aiIdx[pCur->iPage-1], - pCur->apPage[pCur->iPage]->pgno - ); - releasePage(pCur->apPage[pCur->iPage]); - pCur->iPage--; - pCur->info.nSize = 0; - pCur->validNKey = 0; +static Pgno btreePagecount(BtShared *pBt){ + return pBt->nPage; +} +SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ + assert( sqlite3BtreeHoldsMutex(p) ); + assert( ((p->pBt->nPage)&0x8000000)==0 ); + return (int)btreePagecount(p->pBt); } /* -** Move the cursor to point to the root page of its b-tree structure. -** -** If the table has a virtual root page, then the cursor is moved to point -** to the virtual root page instead of the actual root page. A table has a -** virtual root page when the actual root page contains no cells and a -** single child page. This can only happen with the table rooted at page 1. -** -** If the b-tree structure is empty, the cursor state is set to -** CURSOR_INVALID. Otherwise, the cursor is set to point to the first -** cell located on the root (or virtual root) page and the cursor state -** is set to CURSOR_VALID. +** Get a page from the pager and initialize it. This routine is just a +** convenience wrapper around separate calls to btreeGetPage() and +** btreeInitPage(). ** -** If this function returns successfully, it may be assumed that the -** page-header flags indicate that the [virtual] root-page is the expected -** kind of b-tree page (i.e. if when opening the cursor the caller did not -** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, -** indicating a table b-tree, or if the caller did specify a KeyInfo -** structure the flags byte is set to 0x02 or 0x0A, indicating an index -** b-tree). +** If an error occurs, then the value *ppPage is set to is undefined. It +** may remain unchanged, or it may be set to an invalid value. */ -static int moveToRoot(BtCursor *pCur){ - MemPage *pRoot; - int rc = SQLITE_OK; - Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; - - assert( cursorHoldsMutex(pCur) ); - assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); - assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); - assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); - if( pCur->eState>=CURSOR_REQUIRESEEK ){ - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; - } - sqlite3BtreeClearCursor(pCur); - } +static int getAndInitPage( + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage /* Write the page pointer here */ +){ + int rc; + assert( sqlite3_mutex_held(pBt->mutex) ); - if( pCur->iPage>=0 ){ - int i; - for(i=1; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - } - pCur->iPage = 0; + if( pgno>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); - if( rc!=SQLITE_OK ){ - pCur->eState = CURSOR_INVALID; - return rc; - } - pCur->iPage = 0; - - /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor - ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is - ** NULL, the caller expects a table b-tree. If this is not the case, - ** return an SQLITE_CORRUPT error. */ - assert( pCur->apPage[0]->intKey==1 || pCur->apPage[0]->intKey==0 ); - if( (pCur->pKeyInfo==0)!=pCur->apPage[0]->intKey ){ - return SQLITE_CORRUPT_BKPT; + rc = btreeGetPage(pBt, pgno, ppPage, 0); + if( rc==SQLITE_OK ){ + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } } } - /* Assert that the root page is of the correct type. This must be the - ** case as the call to this function that loaded the root-page (either - ** this call or a previous invocation) would have detected corruption - ** if the assumption were not true, and it is not possible for the flags - ** byte to have been modified while this cursor is holding a reference - ** to the page. */ - pRoot = pCur->apPage[0]; - assert( pRoot->pgno==pCur->pgnoRoot ); - assert( pRoot->isInit && (pCur->pKeyInfo==0)==pRoot->intKey ); - - pCur->aiIdx[0] = 0; - pCur->info.nSize = 0; - pCur->atLast = 0; - pCur->validNKey = 0; - - if( pRoot->nCell==0 && !pRoot->leaf ){ - Pgno subpage; - if( pRoot->pgno!=1 ) return SQLITE_CORRUPT_BKPT; - subpage = get4byte(&pRoot->aData[pRoot->hdrOffset+8]); - pCur->eState = CURSOR_VALID; - rc = moveToChild(pCur, subpage); - }else{ - pCur->eState = ((pRoot->nCell>0)?CURSOR_VALID:CURSOR_INVALID); - } + testcase( pgno==0 ); + assert( pgno!=0 || rc==SQLITE_CORRUPT ); return rc; } /* -** Move the cursor down to the left-most leaf entry beneath the -** entry to which it is currently pointing. -** -** The left-most leaf is the one with the smallest key - the first -** in ascending order. +** Release a MemPage. This should be called once for each prior +** call to btreeGetPage. */ -static int moveToLeftmost(BtCursor *pCur){ - Pgno pgno; - int rc = SQLITE_OK; - MemPage *pPage; - - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ - assert( pCur->aiIdx[pCur->iPage]nCell ); - pgno = get4byte(findCell(pPage, pCur->aiIdx[pCur->iPage])); - rc = moveToChild(pCur, pgno); +static void releasePage(MemPage *pPage){ + if( pPage ){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnref(pPage->pDbPage); } - return rc; } /* -** Move the cursor down to the right-most leaf entry beneath the -** page to which it is currently pointing. Notice the difference -** between moveToLeftmost() and moveToRightmost(). moveToLeftmost() -** finds the left-most entry beneath the *entry* whereas moveToRightmost() -** finds the right-most entry beneath the *page*. +** During a rollback, when the pager reloads information into the cache +** so that the cache is restored to its original state at the start of +** the transaction, for each page restored this routine is called. ** -** The right-most entry is the one with the largest key - the last -** key in ascending order. -*/ -static int moveToRightmost(BtCursor *pCur){ - Pgno pgno; - int rc = SQLITE_OK; - MemPage *pPage = 0; - - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ - pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - pCur->aiIdx[pCur->iPage] = pPage->nCell; - rc = moveToChild(pCur, pgno); - } - if( rc==SQLITE_OK ){ - pCur->aiIdx[pCur->iPage] = pPage->nCell-1; - pCur->info.nSize = 0; - pCur->validNKey = 0; - } - return rc; -} - -/* Move the cursor to the first entry in the table. Return SQLITE_OK -** on success. Set *pRes to 0 if the cursor actually points to something -** or set *pRes to 1 if the table is empty. +** This routine needs to reset the extra data section at the end of the +** page to agree with the restored data. */ -SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ - int rc; - - assert( cursorHoldsMutex(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - rc = moveToRoot(pCur); - if( rc==SQLITE_OK ){ - if( pCur->eState==CURSOR_INVALID ){ - assert( pCur->apPage[pCur->iPage]->nCell==0 ); - *pRes = 1; - }else{ - assert( pCur->apPage[pCur->iPage]->nCell>0 ); - *pRes = 0; - rc = moveToLeftmost(pCur); +static void pageReinit(DbPage *pData){ + MemPage *pPage; + pPage = (MemPage *)sqlite3PagerGetExtra(pData); + assert( sqlite3PagerPageRefcount(pData)>0 ); + if( pPage->isInit ){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->isInit = 0; + if( sqlite3PagerPageRefcount(pData)>1 ){ + /* pPage might not be a btree page; it might be an overflow page + ** or ptrmap page or a free page. In those cases, the following + ** call to btreeInitPage() will likely return SQLITE_CORRUPT. + ** But no harm is done by this. And it is very important that + ** btreeInitPage() be called on every btree page so we make + ** the call for every page that comes in for re-initing. */ + btreeInitPage(pPage); } } - return rc; } -/* Move the cursor to the last entry in the table. Return SQLITE_OK -** on success. Set *pRes to 0 if the cursor actually points to something -** or set *pRes to 1 if the table is empty. +/* +** Invoke the busy handler for a btree. */ -SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ - int rc; - - assert( cursorHoldsMutex(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - - /* If the cursor already points to the last entry, this is a no-op. */ - if( CURSOR_VALID==pCur->eState && pCur->atLast ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->aiIdx[pCur->iPage]==pCur->apPage[pCur->iPage]->nCell-1 ); - assert( pCur->apPage[pCur->iPage]->leaf ); -#endif - return SQLITE_OK; - } - - rc = moveToRoot(pCur); - if( rc==SQLITE_OK ){ - if( CURSOR_INVALID==pCur->eState ){ - assert( pCur->apPage[pCur->iPage]->nCell==0 ); - *pRes = 1; - }else{ - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); - pCur->atLast = rc==SQLITE_OK ?1:0; - } - } - return rc; +static int btreeInvokeBusyHandler(void *pArg){ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); + return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); } -/* Move the cursor so that it points to an entry near the key -** specified by pIdxKey or intKey. Return a success code. -** -** For INTKEY tables, the intKey parameter is used. pIdxKey -** must be NULL. For index tables, pIdxKey is used and intKey -** is ignored. -** -** If an exact match is not found, then the cursor is always -** left pointing at a leaf page which would hold the entry if it -** were present. The cursor might point to an entry that comes -** before or after the key. -** -** An integer is written into *pRes which is the result of -** comparing the key with the entry to which the cursor is -** pointing. The meaning of the integer written into -** *pRes is as follows: -** -** *pRes<0 The cursor is left pointing at an entry that -** is smaller than intKey/pIdxKey or if the table is empty -** and the cursor is therefore left point to nothing. +/* +** Open a database file. +** +** zFilename is the name of the database file. If zFilename is NULL +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. ** -** *pRes==0 The cursor is left pointing at an entry that -** exactly matches intKey/pIdxKey. +** If zFilename is ":memory:" then an in-memory database is created +** that is automatically destroyed when it is closed. ** -** *pRes>0 The cursor is left pointing at an entry that -** is larger than intKey/pIdxKey. +** The "flags" parameter is a bitmask that might contain bits +** BTREE_OMIT_JOURNAL and/or BTREE_NO_READLOCK. The BTREE_NO_READLOCK +** bit is also set if the SQLITE_NoReadlock flags is set in db->flags. +** These flags are passed through into sqlite3PagerOpen() and must +** be the same values as PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK. ** +** If the database is already opened in the same database connection +** and we are in shared cache mode, then the open will fail with an +** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared +** objects in the same database connection since doing so will lead +** to problems with locking. */ -SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( - BtCursor *pCur, /* The cursor to be moved */ - UnpackedRecord *pIdxKey, /* Unpacked index key */ - i64 intKey, /* The table key */ - int biasRight, /* If true, bias the search to the high end */ - int *pRes /* Write search results here */ +SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use for this b-tree */ + const char *zFilename, /* Name of the file containing the BTree database */ + sqlite3 *db, /* Associated database handle */ + Btree **ppBtree, /* Pointer to new Btree object written here */ + int flags, /* Options */ + int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ ){ - int rc; + BtShared *pBt = 0; /* Shared part of btree structure */ + Btree *p; /* Handle to return */ + sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ + int rc = SQLITE_OK; /* Result code from this function */ + u8 nReserve; /* Byte of unused space on each page */ + unsigned char zDbHeader[100]; /* Database header content */ - assert( cursorHoldsMutex(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert( pRes ); - assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; - /* If the cursor is already positioned at the point we are trying - ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && pCur->validNKey - && pCur->apPage[0]->intKey - ){ - if( pCur->info.nKey==intKey ){ - *pRes = 0; - return SQLITE_OK; - } - if( pCur->atLast && pCur->info.nKeymutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( db->flags & SQLITE_NoReadlock ){ + flags |= BTREE_NO_READLOCK; } - assert( pCur->apPage[pCur->iPage] ); - assert( pCur->apPage[pCur->iPage]->isInit ); - assert( pCur->apPage[pCur->iPage]->nCell>0 || pCur->eState==CURSOR_INVALID ); - if( pCur->eState==CURSOR_INVALID ){ - *pRes = -1; - assert( pCur->apPage[pCur->iPage]->nCell==0 ); - return SQLITE_OK; + if( isMemdb ){ + flags |= BTREE_MEMORY; } - assert( pCur->apPage[0]->intKey || pIdxKey ); - for(;;){ - int lwr, upr; - Pgno chldPg; - MemPage *pPage = pCur->apPage[pCur->iPage]; - int c; + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } + p = sqlite3MallocZero(sizeof(Btree)); + if( !p ){ + return SQLITE_NOMEM; + } + p->inTrans = TRANS_NONE; + p->db = db; +#ifndef SQLITE_OMIT_SHARED_CACHE + p->lock.pBtree = p; + p->lock.iTable = 1; +#endif - /* pPage->nCell must be greater than zero. If this is the root-page - ** the cursor would have been INVALID above and this for(;;) loop - ** not run. If this is not the root-page, then the moveToChild() routine - ** would have already detected db corruption. Similarly, pPage must - ** be the right kind (index or table) of b-tree page. Otherwise - ** a moveToChild() or moveToRoot() call would have detected corruption. */ - assert( pPage->nCell>0 ); - assert( pPage->intKey==(pIdxKey==0) ); - lwr = 0; - upr = pPage->nCell-1; - if( biasRight ){ - pCur->aiIdx[pCur->iPage] = (u16)upr; +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* + ** If this Btree is a candidate for shared cache, try to find an + ** existing BtShared object that we can share with + */ + if( isMemdb==0 && isTempDb==0 ){ + if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ + int nFullPathname = pVfs->mxPathname+1; + char *zFullPathname = sqlite3Malloc(nFullPathname); + sqlite3_mutex *mutexShared; + p->sharable = 1; + if( !zFullPathname ){ + sqlite3_free(p); + return SQLITE_NOMEM; + } + sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); + mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); + sqlite3_mutex_enter(mutexOpen); + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutexShared); + for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ + assert( pBt->nRef>0 ); + if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager)) + && sqlite3PagerVfs(pBt->pPager)==pVfs ){ + int iDb; + for(iDb=db->nDb-1; iDb>=0; iDb--){ + Btree *pExisting = db->aDb[iDb].pBt; + if( pExisting && pExisting->pBt==pBt ){ + sqlite3_mutex_leave(mutexShared); + sqlite3_mutex_leave(mutexOpen); + sqlite3_free(zFullPathname); + sqlite3_free(p); + return SQLITE_CONSTRAINT; + } + } + p->pBt = pBt; + pBt->nRef++; + break; + } + } + sqlite3_mutex_leave(mutexShared); + sqlite3_free(zFullPathname); + } +#ifdef SQLITE_DEBUG + else{ + /* In debug mode, we mark all persistent databases as sharable + ** even when they are not. This exercises the locking code and + ** gives more opportunity for asserts(sqlite3_mutex_held()) + ** statements to find locking problems. + */ + p->sharable = 1; + } +#endif + } +#endif + if( pBt==0 ){ + /* + ** The following asserts make sure that structures used by the btree are + ** the right size. This is to guard against size changes that result + ** when compiling on a different architecture. + */ + assert( sizeof(i64)==8 || sizeof(i64)==4 ); + assert( sizeof(u64)==8 || sizeof(u64)==4 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(Pgno)==4 ); + + pBt = sqlite3MallocZero( sizeof(*pBt) ); + if( pBt==0 ){ + rc = SQLITE_NOMEM; + goto btree_open_out; + } + rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, + EXTRA_SIZE, flags, vfsFlags, pageReinit); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); + } + if( rc!=SQLITE_OK ){ + goto btree_open_out; + } + pBt->openFlags = (u8)flags; + pBt->db = db; + sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); + p->pBt = pBt; + + pBt->pCursor = 0; + pBt->pPage1 = 0; + pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); +#ifdef SQLITE_SECURE_DELETE + pBt->secureDelete = 1; +#endif + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); + if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE + || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ + pBt->pageSize = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the magic name ":memory:" will create an in-memory database, then + ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if + ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if + ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a + ** regular file-name. In this case the auto-vacuum applies as per normal. + */ + if( zFilename && !isMemdb ){ + pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); + pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); + } +#endif + nReserve = 0; }else{ - pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); + nReserve = zDbHeader[20]; + pBt->pageSizeFixed = 1; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); +#endif } - for(;;){ - int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */ - u8 *pCell; /* Pointer to current cell in pPage */ - - pCur->info.nSize = 0; - pCell = findCell(pPage, idx) + pPage->childPtrSize; - if( pPage->intKey ){ - i64 nCellKey; - if( pPage->hasData ){ - u32 dummy; - pCell += getVarint32(pCell, dummy); - } - getVarint(pCell, (u64*)&nCellKey); - if( nCellKey==intKey ){ - c = 0; - }else if( nCellKeyintKey ); - c = +1; + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + if( rc ) goto btree_open_out; + pBt->usableSize = pBt->pageSize - nReserve; + assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* Add the new BtShared object to the linked list sharable BtShareds. + */ + if( p->sharable ){ + sqlite3_mutex *mutexShared; + pBt->nRef = 1; + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ + pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); + if( pBt->mutex==0 ){ + rc = SQLITE_NOMEM; + db->mallocFailed = 0; + goto btree_open_out; } - pCur->validNKey = 1; - pCur->info.nKey = nCellKey; - }else{ - /* The maximum supported page-size is 65536 bytes. This means that - ** the maximum number of record bytes stored on an index B-Tree - ** page is less than 16384 bytes and may be stored as a 2-byte - ** varint. This information is used to attempt to avoid parsing - ** the entire cell by checking for the cases where the record is - ** stored entirely within the b-tree page by inspecting the first - ** 2 bytes of the cell. - */ - int nCell = pCell[0]; - if( !(nCell & 0x80) && nCell<=pPage->maxLocal ){ - /* This branch runs if the record-size field of the cell is a - ** single byte varint and the record fits entirely on the main - ** b-tree page. */ - c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[1], pIdxKey); - }else if( !(pCell[1] & 0x80) - && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal - ){ - /* The record-size field is a 2 byte varint and the record - ** fits entirely on the main b-tree page. */ - c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + } + sqlite3_mutex_enter(mutexShared); + pBt->pNext = GLOBAL(BtShared*,sqlite3SharedCacheList); + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt; + sqlite3_mutex_leave(mutexShared); + } +#endif + } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* If the new Btree uses a sharable pBtShared, then link the new + ** Btree into the list of all sharable Btrees for the same connection. + ** The list is kept in ascending order by pBt address. + */ + if( p->sharable ){ + int i; + Btree *pSib; + for(i=0; inDb; i++){ + if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ + while( pSib->pPrev ){ pSib = pSib->pPrev; } + if( p->pBtpBt ){ + p->pNext = pSib; + p->pPrev = 0; + pSib->pPrev = p; }else{ - /* The record flows over onto one or more overflow pages. In - ** this case the whole cell needs to be parsed, a buffer allocated - ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. */ - void *pCellKey; - u8 * const pCellBody = pCell - pPage->childPtrSize; - btreeParseCellPtr(pPage, pCellBody, &pCur->info); - nCell = (int)pCur->info.nKey; - pCellKey = sqlite3Malloc( nCell ); - if( pCellKey==0 ){ - rc = SQLITE_NOMEM; - goto moveto_finish; + while( pSib->pNext && pSib->pNext->pBtpBt ){ + pSib = pSib->pNext; } - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); - if( rc ){ - sqlite3_free(pCellKey); - goto moveto_finish; + p->pNext = pSib->pNext; + p->pPrev = pSib; + if( p->pNext ){ + p->pNext->pPrev = p; } - c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); - sqlite3_free(pCellKey); - } - } - if( c==0 ){ - if( pPage->intKey && !pPage->leaf ){ - lwr = idx; - upr = lwr - 1; - break; - }else{ - *pRes = 0; - rc = SQLITE_OK; - goto moveto_finish; + pSib->pNext = p; } - } - if( c<0 ){ - lwr = idx+1; - }else{ - upr = idx-1; - } - if( lwr>upr ){ break; } - pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); } - assert( lwr==upr+1 ); - assert( pPage->isInit ); - if( pPage->leaf ){ - chldPg = 0; - }else if( lwr>=pPage->nCell ){ - chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); - }else{ - chldPg = get4byte(findCell(pPage, lwr)); + } +#endif + *ppBtree = p; + +btree_open_out: + if( rc!=SQLITE_OK ){ + if( pBt && pBt->pPager ){ + sqlite3PagerClose(pBt->pPager); } - if( chldPg==0 ){ - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - *pRes = c; - rc = SQLITE_OK; - goto moveto_finish; + sqlite3_free(pBt); + sqlite3_free(p); + *ppBtree = 0; + }else{ + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); } - pCur->aiIdx[pCur->iPage] = (u16)lwr; - pCur->info.nSize = 0; - pCur->validNKey = 0; - rc = moveToChild(pCur, chldPg); - if( rc ) goto moveto_finish; } -moveto_finish: + if( mutexOpen ){ + assert( sqlite3_mutex_held(mutexOpen) ); + sqlite3_mutex_leave(mutexOpen); + } return rc; } - -/* -** Return TRUE if the cursor is not pointing at an entry of the table. -** -** TRUE will be returned after a call to sqlite3BtreeNext() moves -** past the last entry in the table or sqlite3BtreePrev() moves past -** the first entry. TRUE is also returned if the table is empty. -*/ -SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ - /* TODO: What if the cursor is in CURSOR_REQUIRESEEK but all table entries - ** have been deleted? This API will need to change to return an error code - ** as well as the boolean result value. - */ - return (CURSOR_VALID!=pCur->eState); -} - /* -** Advance the cursor to the next entry in the database. If -** successful then set *pRes=0. If the cursor -** was already pointing to the last entry in the database before -** this routine was called, then set *pRes=1. +** Decrement the BtShared.nRef counter. When it reaches zero, +** remove the BtShared structure from the sharing list. Return +** true if the BtShared.nRef counter reaches zero and return +** false if it is still positive. */ -SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ - int rc; - int idx; - MemPage *pPage; - - assert( cursorHoldsMutex(pCur) ); - rc = restoreCursorPosition(pCur); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pRes!=0 ); - if( CURSOR_INVALID==pCur->eState ){ - *pRes = 1; - return SQLITE_OK; - } - if( pCur->skipNext>0 ){ - pCur->skipNext = 0; - *pRes = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; - - pPage = pCur->apPage[pCur->iPage]; - idx = ++pCur->aiIdx[pCur->iPage]; - assert( pPage->isInit ); - assert( idx<=pPage->nCell ); +static int removeFromSharingList(BtShared *pBt){ +#ifndef SQLITE_OMIT_SHARED_CACHE + sqlite3_mutex *pMaster; + BtShared *pList; + int removed = 0; - pCur->info.nSize = 0; - pCur->validNKey = 0; - if( idx>=pPage->nCell ){ - if( !pPage->leaf ){ - rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); - if( rc ) return rc; - rc = moveToLeftmost(pCur); - *pRes = 0; - return rc; - } - do{ - if( pCur->iPage==0 ){ - *pRes = 1; - pCur->eState = CURSOR_INVALID; - return SQLITE_OK; - } - moveToParent(pCur); - pPage = pCur->apPage[pCur->iPage]; - }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); - *pRes = 0; - if( pPage->intKey ){ - rc = sqlite3BtreeNext(pCur, pRes); + assert( sqlite3_mutex_notheld(pBt->mutex) ); + pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(pMaster); + pBt->nRef--; + if( pBt->nRef<=0 ){ + if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt->pNext; }else{ - rc = SQLITE_OK; + pList = GLOBAL(BtShared*,sqlite3SharedCacheList); + while( ALWAYS(pList) && pList->pNext!=pBt ){ + pList=pList->pNext; + } + if( ALWAYS(pList) ){ + pList->pNext = pBt->pNext; + } } - return rc; + if( SQLITE_THREADSAFE ){ + sqlite3_mutex_free(pBt->mutex); + } + removed = 1; } - *pRes = 0; - if( pPage->leaf ){ - return SQLITE_OK; + sqlite3_mutex_leave(pMaster); + return removed; +#else + return 1; +#endif +} + +/* +** Make sure pBt->pTmpSpace points to an allocation of +** MX_CELL_SIZE(pBt) bytes. +*/ +static void allocateTempSpace(BtShared *pBt){ + if( !pBt->pTmpSpace ){ + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); } - rc = moveToLeftmost(pCur); - return rc; } +/* +** Free the pBt->pTmpSpace allocation +*/ +static void freeTempSpace(BtShared *pBt){ + sqlite3PageFree( pBt->pTmpSpace); + pBt->pTmpSpace = 0; +} /* -** Step the cursor to the back to the previous entry in the database. If -** successful then set *pRes=0. If the cursor -** was already pointing to the first entry in the database before -** this routine was called, then set *pRes=1. +** Close an open database and invalidate all cursors. */ -SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ - int rc; - MemPage *pPage; +SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ + BtShared *pBt = p->pBt; + BtCursor *pCur; - assert( cursorHoldsMutex(pCur) ); - rc = restoreCursorPosition(pCur); - if( rc!=SQLITE_OK ){ - return rc; - } - pCur->atLast = 0; - if( CURSOR_INVALID==pCur->eState ){ - *pRes = 1; - return SQLITE_OK; - } - if( pCur->skipNext<0 ){ - pCur->skipNext = 0; - *pRes = 0; - return SQLITE_OK; + /* Close all cursors opened via this handle. */ + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + pCur = pBt->pCursor; + while( pCur ){ + BtCursor *pTmp = pCur; + pCur = pCur->pNext; + if( pTmp->pBtree==p ){ + sqlite3BtreeCloseCursor(pTmp); + } } - pCur->skipNext = 0; - pPage = pCur->apPage[pCur->iPage]; - assert( pPage->isInit ); - if( !pPage->leaf ){ - int idx = pCur->aiIdx[pCur->iPage]; - rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); - if( rc ){ - return rc; - } - rc = moveToRightmost(pCur); - }else{ - while( pCur->aiIdx[pCur->iPage]==0 ){ - if( pCur->iPage==0 ){ - pCur->eState = CURSOR_INVALID; - *pRes = 1; - return SQLITE_OK; - } - moveToParent(pCur); - } - pCur->info.nSize = 0; - pCur->validNKey = 0; + /* Rollback any active transaction and free the handle structure. + ** The call to sqlite3BtreeRollback() drops any table-locks held by + ** this handle. + */ + sqlite3BtreeRollback(p); + sqlite3BtreeLeave(p); - pCur->aiIdx[pCur->iPage]--; - pPage = pCur->apPage[pCur->iPage]; - if( pPage->intKey && !pPage->leaf ){ - rc = sqlite3BtreePrevious(pCur, pRes); - }else{ - rc = SQLITE_OK; + /* If there are still other outstanding references to the shared-btree + ** structure, return now. The remainder of this procedure cleans + ** up the shared-btree. + */ + assert( p->wantToLock==0 && p->locked==0 ); + if( !p->sharable || removeFromSharingList(pBt) ){ + /* The pBt is no longer on the sharing list, so we can access + ** it without having to hold the mutex. + ** + ** Clean out and delete the BtShared object. + */ + assert( !pBt->pCursor ); + sqlite3PagerClose(pBt->pPager); + if( pBt->xFreeSchema && pBt->pSchema ){ + pBt->xFreeSchema(pBt->pSchema); } + sqlite3DbFree(0, pBt->pSchema); + freeTempSpace(pBt); + sqlite3_free(pBt); } - *pRes = 0; + +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( p->wantToLock==0 ); + assert( p->locked==0 ); + if( p->pPrev ) p->pPrev->pNext = p->pNext; + if( p->pNext ) p->pNext->pPrev = p->pPrev; +#endif + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Change the limit on the number of pages allowed in the cache. +** +** The maximum number of cache pages is set to the absolute +** value of mxPage. If mxPage is negative, the pager will +** operate asynchronously - it will not stop to do fsync()s +** to insure data is written to the disk surface before +** continuing. Transactions still work if synchronous is off, +** and the database cannot be corrupted if this program +** crashes. But if the operating system crashes or there is +** an abrupt power failure when synchronous is off, the database +** could be left in an inconsistent and unrecoverable state. +** Synchronous is on by default so database corruption is not +** normally a worry. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetCachesize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* +** Change the way data is synced to disk in order to increase or decrease +** how well the database resists damage due to OS crashes and power +** failures. Level 1 is the same as asynchronous (no syncs() occur and +** there is a high probability of damage) Level 2 is the default. There +** is a very low but non-zero probability of damage. Level 3 reduces the +** probability of damage to near zero but with a write performance reduction. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel( + Btree *p, /* The btree to set the safety level on */ + int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ + int fullSync, /* PRAGMA fullfsync. */ + int ckptFullSync /* PRAGMA checkpoint_fullfync */ +){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + assert( level>=1 && level<=3 ); + sqlite3BtreeEnter(p); + sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync, ckptFullSync); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif + +/* +** Return TRUE if the given btree is set to safety level 1. In other +** words, return TRUE if no sync() occurs on the disk files. +*/ +SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree *p){ + BtShared *pBt = p->pBt; + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + assert( pBt && pBt->pPager ); + rc = sqlite3PagerNosync(pBt->pPager); + sqlite3BtreeLeave(p); return rc; } /* -** Allocate a new page from the database file. +** Change the default pages size and the number of reserved bytes per page. +** Or, if the page size has already been fixed, return SQLITE_READONLY +** without changing anything. ** -** The new page is marked as dirty. (In other words, sqlite3PagerWrite() -** has already been called on the new page.) The new page has also -** been referenced and the calling routine is responsible for calling -** sqlite3PagerUnref() on the new page when it is done. +** The page size must be a power of 2 between 512 and 65536. If the page +** size supplied does not meet this constraint then the page size is not +** changed. ** -** SQLITE_OK is returned on success. Any other return value indicates -** an error. *ppPage and *pPgno are undefined in the event of an error. -** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. +** Page sizes are constrained to be a power of two so that the region +** of the database file used for locking (beginning at PENDING_BYTE, +** the first byte past the 1GB boundary, 0x40000000) needs to occur +** at the beginning of a page. ** -** If the "nearby" parameter is not 0, then a (feeble) effort is made to -** locate a page close to the page number "nearby". This can be used in an -** attempt to keep related pages close to each other in the database file, -** which in turn can make database access faster. +** If parameter nReserve is less than zero, then the number of reserved +** bytes per page is left unchanged. ** -** If the "exact" parameter is not 0, and the page-number nearby exists -** anywhere on the free-list, then it is guarenteed to be returned. This -** is only used by auto-vacuum databases when allocating a new table. +** If the iFix!=0 then the pageSizeFixed flag is set so that the page size +** and autovacuum mode can no longer be changed. */ -static int allocateBtreePage( - BtShared *pBt, - MemPage **ppPage, - Pgno *pPgno, - Pgno nearby, - u8 exact -){ - MemPage *pPage1; +SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ + int rc = SQLITE_OK; + BtShared *pBt = p->pBt; + assert( nReserve>=-1 && nReserve<=255 ); + sqlite3BtreeEnter(p); + if( pBt->pageSizeFixed ){ + sqlite3BtreeLeave(p); + return SQLITE_READONLY; + } + if( nReserve<0 ){ + nReserve = pBt->pageSize - pBt->usableSize; + } + assert( nReserve>=0 && nReserve<=255 ); + if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && + ((pageSize-1)&pageSize)==0 ){ + assert( (pageSize & 7)==0 ); + assert( !pBt->pPage1 && !pBt->pCursor ); + pBt->pageSize = (u32)pageSize; + freeTempSpace(pBt); + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + pBt->usableSize = pBt->pageSize - (u16)nReserve; + if( iFix ) pBt->pageSizeFixed = 1; + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Return the currently defined page size +*/ +SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ + return p->pBt->pageSize; +} + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) +/* +** Return the number of bytes of space at the end of every page that +** are intentually left unused. This is the "reserved" space that is +** sometimes used by extensions. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetReserve(Btree *p){ + int n; + sqlite3BtreeEnter(p); + n = p->pBt->pageSize - p->pBt->usableSize; + sqlite3BtreeLeave(p); + return n; +} + +/* +** Set the maximum page count for a database if mxPage is positive. +** No changes are made if mxPage is 0 or negative. +** Regardless of the value of mxPage, return the maximum page count. +*/ +SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ + int n; + sqlite3BtreeEnter(p); + n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return n; +} + +/* +** Set the secureDelete flag if newFlag is 0 or 1. If newFlag is -1, +** then make no changes. Always return the value of the secureDelete +** setting after the change. +*/ +SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ + int b; + if( p==0 ) return 0; + sqlite3BtreeEnter(p); + if( newFlag>=0 ){ + p->pBt->secureDelete = (newFlag!=0) ? 1 : 0; + } + b = p->pBt->secureDelete; + sqlite3BtreeLeave(p); + return b; +} +#endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ + +/* +** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' +** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it +** is disabled. The default value for the auto-vacuum property is +** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return SQLITE_READONLY; +#else + BtShared *pBt = p->pBt; + int rc = SQLITE_OK; + u8 av = (u8)autoVacuum; + + sqlite3BtreeEnter(p); + if( pBt->pageSizeFixed && (av ?1:0)!=pBt->autoVacuum ){ + rc = SQLITE_READONLY; + }else{ + pBt->autoVacuum = av ?1:0; + pBt->incrVacuum = av==2 ?1:0; + } + sqlite3BtreeLeave(p); + return rc; +#endif +} + +/* +** Return the value of the 'auto-vacuum' property. If auto-vacuum is +** enabled 1 is returned. Otherwise 0. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return BTREE_AUTOVACUUM_NONE; +#else int rc; - u32 n; /* Number of pages on the freelist */ - u32 k; /* Number of leaves on the trunk of the freelist */ - MemPage *pTrunk = 0; - MemPage *pPrevTrunk = 0; - Pgno mxPage; /* Total size of the database file */ + sqlite3BtreeEnter(p); + rc = ( + (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: + (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: + BTREE_AUTOVACUUM_INCR + ); + sqlite3BtreeLeave(p); + return rc; +#endif +} + + +/* +** Get a reference to pPage1 of the database file. This will +** also acquire a readlock on that file. +** +** SQLITE_OK is returned on success. If the file is not a +** well-formed database file, then SQLITE_CORRUPT is returned. +** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM +** is returned if we run out of memory. +*/ +static int lockBtree(BtShared *pBt){ + int rc; /* Result code from subfunctions */ + MemPage *pPage1; /* Page 1 of the database file */ + int nPage; /* Number of pages in the database */ + int nPageFile = 0; /* Number of pages in the database file */ + int nPageHeader; /* Number of pages in the database according to hdr */ assert( sqlite3_mutex_held(pBt->mutex) ); - pPage1 = pBt->pPage1; - mxPage = btreePagecount(pBt); - n = get4byte(&pPage1->aData[36]); - testcase( n==mxPage-1 ); - if( n>=mxPage ){ - return SQLITE_CORRUPT_BKPT; + assert( pBt->pPage1==0 ); + rc = sqlite3PagerSharedLock(pBt->pPager); + if( rc!=SQLITE_OK ) return rc; + rc = btreeGetPage(pBt, 1, &pPage1, 0); + if( rc!=SQLITE_OK ) return rc; + + /* Do some checking to help insure the file we opened really is + ** a valid database file. + */ + nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); + sqlite3PagerPagecount(pBt->pPager, &nPageFile); + if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ + nPage = nPageFile; } - if( n>0 ){ - /* There are pages on the freelist. Reuse one of those pages. */ - Pgno iTrunk; - u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ - - /* If the 'exact' parameter was true and a query of the pointer-map - ** shows that the page 'nearby' is somewhere on the free-list, then - ** the entire-list will be searched for that page. + if( nPage>0 ){ + u32 pageSize; + u32 usableSize; + u8 *page1 = pPage1->aData; + rc = SQLITE_NOTADB; + if( memcmp(page1, zMagicHeader, 16)!=0 ){ + goto page1_init_failed; + } + +#ifdef SQLITE_OMIT_WAL + if( page1[18]>1 ){ + pBt->readOnly = 1; + } + if( page1[19]>1 ){ + goto page1_init_failed; + } +#else + if( page1[18]>2 ){ + pBt->readOnly = 1; + } + if( page1[19]>2 ){ + goto page1_init_failed; + } + + /* If the write version is set to 2, this database should be accessed + ** in WAL mode. If the log is not already open, open it now. Then + ** return SQLITE_OK and return without populating BtShared.pPage1. + ** The caller detects this and calls this function again. This is + ** required as the version of page 1 currently in the page1 buffer + ** may not be the latest version - there may be a newer one in the log + ** file. */ -#ifndef SQLITE_OMIT_AUTOVACUUM - if( exact && nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; + if( page1[19]==2 && pBt->doNotUseWAL==0 ){ + int isOpen = 0; + rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); + if( rc!=SQLITE_OK ){ + goto page1_init_failed; + }else if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; } - *pPgno = nearby; + rc = SQLITE_NOTADB; } #endif - /* Decrement the free-list count by 1. Set iTrunk to the index of the - ** first free-list trunk page. iPrevTrunk is initially 1. + /* The maximum embedded fraction must be exactly 25%. And the minimum + ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + ** The original design allowed these amounts to vary, but as of + ** version 3.6.0, we require them to be fixed. */ - rc = sqlite3PagerWrite(pPage1->pDbPage); - if( rc ) return rc; - put4byte(&pPage1->aData[36], n-1); + if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ + goto page1_init_failed; + } + pageSize = (page1[16]<<8) | (page1[17]<<16); + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 + ){ + goto page1_init_failed; + } + assert( (pageSize & 7)==0 ); + usableSize = pageSize - page1[20]; + if( (u32)pageSize!=pBt->pageSize ){ + /* After reading the first page of the database assuming a page size + ** of BtShared.pageSize, we have discovered that the page-size is + ** actually pageSize. Unlock the database, leave pBt->pPage1 at + ** zero and return SQLITE_OK. The caller will call this function + ** again with the correct page-size. + */ + releasePage(pPage1); + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; + freeTempSpace(pBt); + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, + pageSize-usableSize); + return rc; + } + if( (pBt->db->flags & SQLITE_RecoveryMode)==0 && nPage>nPageFile ){ + rc = SQLITE_CORRUPT_BKPT; + goto page1_init_failed; + } + if( usableSize<480 ){ + goto page1_init_failed; + } + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); +#endif + } - /* The code within this loop is run only once if the 'searchList' variable - ** is not true. Otherwise, it runs once for each trunk-page on the - ** free-list until the page 'nearby' is located. - */ - do { - pPrevTrunk = pTrunk; - if( pPrevTrunk ){ - iTrunk = get4byte(&pPrevTrunk->aData[0]); - }else{ - iTrunk = get4byte(&pPage1->aData[32]); - } - testcase( iTrunk==mxPage ); - if( iTrunk>mxPage ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); - } - if( rc ){ - pTrunk = 0; - goto end_allocate_page; - } + /* maxLocal is the maximum amount of payload to store locally for + ** a cell. Make sure it is small enough so that at least minFanout + ** cells can will fit on one page. We assume a 10-byte page header. + ** Besides the payload, the cell must store: + ** 2-byte pointer to the cell + ** 4-byte child pointer + ** 9-byte nKey value + ** 4-byte nData value + ** 4-byte overflow page pointer + ** So a cell consists of a 2-byte pointer, a header which is as much as + ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow + ** page pointer. + */ + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); + assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); + pBt->pPage1 = pPage1; + pBt->nPage = nPage; + return SQLITE_OK; - k = get4byte(&pTrunk->aData[4]); - if( k==0 && !searchList ){ - /* The trunk has no leaves and the list is not being searched. - ** So extract the trunk page itself and use it as the newly - ** allocated page */ - assert( pPrevTrunk==0 ); - rc = sqlite3PagerWrite(pTrunk->pDbPage); - if( rc ){ - goto end_allocate_page; - } - *pPgno = iTrunk; - memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); - *ppPage = pTrunk; - pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); - }else if( k>(u32)(pBt->usableSize/4 - 2) ){ - /* Value of k is out of range. Database corruption */ - rc = SQLITE_CORRUPT_BKPT; - goto end_allocate_page; -#ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList && nearby==iTrunk ){ - /* The list is being searched and this trunk page is the page - ** to allocate, regardless of whether it has leaves. - */ - assert( *pPgno==iTrunk ); - *ppPage = pTrunk; - searchList = 0; - rc = sqlite3PagerWrite(pTrunk->pDbPage); - if( rc ){ - goto end_allocate_page; - } - if( k==0 ){ - if( !pPrevTrunk ){ - memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); - }else{ - rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); - if( rc!=SQLITE_OK ){ - goto end_allocate_page; - } - memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); - } - }else{ - /* The trunk page is required by the caller but it contains - ** pointers to free-list leaves. The first leaf becomes a trunk - ** page in this case. - */ - MemPage *pNewTrunk; - Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); - if( iNewTrunk>mxPage ){ - rc = SQLITE_CORRUPT_BKPT; - goto end_allocate_page; - } - testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); - if( rc!=SQLITE_OK ){ - goto end_allocate_page; - } - rc = sqlite3PagerWrite(pNewTrunk->pDbPage); - if( rc!=SQLITE_OK ){ - releasePage(pNewTrunk); - goto end_allocate_page; - } - memcpy(&pNewTrunk->aData[0], &pTrunk->aData[0], 4); - put4byte(&pNewTrunk->aData[4], k-1); - memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); - releasePage(pNewTrunk); - if( !pPrevTrunk ){ - assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); - put4byte(&pPage1->aData[32], iNewTrunk); - }else{ - rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); - if( rc ){ - goto end_allocate_page; - } - put4byte(&pPrevTrunk->aData[0], iNewTrunk); - } - } - pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); -#endif - }else if( k>0 ){ - /* Extract a leaf from the trunk */ - u32 closest; - Pgno iPage; - unsigned char *aData = pTrunk->aData; - rc = sqlite3PagerWrite(pTrunk->pDbPage); - if( rc ){ - goto end_allocate_page; - } - if( nearby>0 ){ - u32 i; - int dist; - closest = 0; - dist = get4byte(&aData[8]) - nearby; - if( dist<0 ) dist = -dist; - for(i=1; imxPage ){ - rc = SQLITE_CORRUPT_BKPT; - goto end_allocate_page; - } - testcase( iPage==mxPage ); - if( !searchList || iPage==nearby ){ - int noContent; - *pPgno = iPage; - TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" - ": %d more free pages\n", - *pPgno, closest+1, k, pTrunk->pgno, n-1)); - if( closestpDbPage) ); - noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite((*ppPage)->pDbPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - } - } - searchList = 0; - } - } - releasePage(pPrevTrunk); - pPrevTrunk = 0; - }while( searchList ); - }else{ - /* There are no pages on the freelist, so create a new page at the - ** end of the file */ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - if( rc ) return rc; - pBt->nPage++; - if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; - -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, pBt->nPage) ){ - /* If *pPgno refers to a pointer-map page, allocate two new pages - ** at the end of the file instead of one. The first allocated page - ** becomes a new pointer-map page, the second is used by the caller. - */ - MemPage *pPg = 0; - TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); - assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pPg->pDbPage); - releasePage(pPg); - } - if( rc ) return rc; - pBt->nPage++; - if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ){ pBt->nPage++; } - } -#endif - put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage); - *pPgno = pBt->nPage; +page1_init_failed: + releasePage(pPage1); + pBt->pPage1 = 0; + return rc; +} - assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, 1); - if( rc ) return rc; - rc = sqlite3PagerWrite((*ppPage)->pDbPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - } - TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); +/* +** If there are no outstanding cursors and we are not in the middle +** of a transaction but there is a read lock on the database, then +** this routine unrefs the first page of the database file which +** has the effect of releasing the read lock. +** +** If there is a transaction in progress, this routine is a no-op. +*/ +static void unlockBtreeIfUnused(BtShared *pBt){ + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pBt->pCursor==0 || pBt->inTransaction>TRANS_NONE ); + if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ + assert( pBt->pPage1->aData ); + assert( sqlite3PagerRefcount(pBt->pPager)==1 ); + assert( pBt->pPage1->aData ); + releasePage(pBt->pPage1); + pBt->pPage1 = 0; } +} - assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); +/* +** If pBt points to an empty file then convert that empty file +** into a new empty database by initializing the first page of +** the database. +*/ +static int newDatabase(BtShared *pBt){ + MemPage *pP1; + unsigned char *data; + int rc; -end_allocate_page: - releasePage(pTrunk); - releasePage(pPrevTrunk); - if( rc==SQLITE_OK ){ - if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ - releasePage(*ppPage); - return SQLITE_CORRUPT_BKPT; - } - (*ppPage)->isInit = 0; - }else{ - *ppPage = 0; + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->nPage>0 ){ + return SQLITE_OK; } - return rc; + pP1 = pBt->pPage1; + assert( pP1!=0 ); + data = pP1->aData; + rc = sqlite3PagerWrite(pP1->pDbPage); + if( rc ) return rc; + memcpy(data, zMagicHeader, sizeof(zMagicHeader)); + assert( sizeof(zMagicHeader)==16 ); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); + data[18] = 1; + data[19] = 1; + assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); + data[20] = (u8)(pBt->pageSize - pBt->usableSize); + data[21] = 64; + data[22] = 32; + data[23] = 32; + memset(&data[24], 0, 100-24); + zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); + pBt->pageSizeFixed = 1; +#ifndef SQLITE_OMIT_AUTOVACUUM + assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); + assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 ); + put4byte(&data[36 + 4*4], pBt->autoVacuum); + put4byte(&data[36 + 7*4], pBt->incrVacuum); +#endif + pBt->nPage = 1; + data[31] = 1; + return SQLITE_OK; } /* -** This function is used to add page iPage to the database file free-list. -** It is assumed that the page is not already a part of the free-list. +** Attempt to start a new transaction. A write-transaction +** is started if the second argument is nonzero, otherwise a read- +** transaction. If the second argument is 2 or more and exclusive +** transaction is started, meaning that no other process is allowed +** to access the database. A preexisting transaction may not be +** upgraded to exclusive by calling this routine a second time - the +** exclusivity flag only works for a new transaction. ** -** The value passed as the second argument to this function is optional. -** If the caller happens to have a pointer to the MemPage object -** corresponding to page iPage handy, it may pass it as the second value. -** Otherwise, it may pass NULL. +** A write-transaction must be started before attempting any +** changes to the database. None of the following routines +** will work unless a transaction is started first: ** -** If a pointer to a MemPage object is passed as the second argument, -** its reference count is not altered by this function. +** sqlite3BtreeCreateTable() +** sqlite3BtreeCreateIndex() +** sqlite3BtreeClearTable() +** sqlite3BtreeDropTable() +** sqlite3BtreeInsert() +** sqlite3BtreeDelete() +** sqlite3BtreeUpdateMeta() +** +** If an initial attempt to acquire the lock fails because of lock contention +** and the database was previously unlocked, then invoke the busy handler +** if there is one. But if there was previously a read-lock, do not +** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is +** returned when there is already a read-lock in order to avoid a deadlock. +** +** Suppose there are two processes A and B. A has a read lock and B has +** a reserved lock. B tries to promote to exclusive but is blocked because +** of A's read lock. A tries to promote to reserved but is blocked by B. +** One or the other of the two processes must give way or there can be +** no progress. By returning SQLITE_BUSY and not invoking the busy callback +** when A already has a read lock, we encourage A to give up and let B +** proceed. */ -static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ - MemPage *pTrunk = 0; /* Free-list trunk page */ - Pgno iTrunk = 0; /* Page number of free-list trunk page */ - MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ - MemPage *pPage; /* Page being freed. May be NULL. */ - int rc; /* Return Code */ - int nFree; /* Initial number of pages on free-list */ +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ + sqlite3 *pBlock = 0; + BtShared *pBt = p->pBt; + int rc = SQLITE_OK; - assert( sqlite3_mutex_held(pBt->mutex) ); - assert( iPage>1 ); - assert( !pMemPage || pMemPage->pgno==iPage ); + sqlite3BtreeEnter(p); + btreeIntegrity(p); - if( pMemPage ){ - pPage = pMemPage; - sqlite3PagerRef(pPage->pDbPage); - }else{ - pPage = btreePageLookup(pBt, iPage); + /* If the btree is already in a write-transaction, or it + ** is already in a read-transaction and a read-transaction + ** is requested, this is a no-op. + */ + if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ + goto trans_begun; } - /* Increment the free page count on pPage1 */ - rc = sqlite3PagerWrite(pPage1->pDbPage); - if( rc ) goto freepage_out; - nFree = get4byte(&pPage1->aData[36]); - put4byte(&pPage1->aData[36], nFree+1); - - if( pBt->secureDelete ){ - /* If the secure_delete option is enabled, then - ** always fully overwrite deleted information with zeros. - */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) - || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) - ){ - goto freepage_out; - } - memset(pPage->aData, 0, pPage->pBt->pageSize); + /* Write transactions are not possible on a read-only database */ + if( pBt->readOnly && wrflag ){ + rc = SQLITE_READONLY; + goto trans_begun; } - /* If the database supports auto-vacuum, write an entry in the pointer-map - ** to indicate that the page is free. +#ifndef SQLITE_OMIT_SHARED_CACHE + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. */ - if( ISAUTOVACUUM ){ - ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); - if( rc ) goto freepage_out; + if( (wrflag && pBt->inTransaction==TRANS_WRITE) || pBt->isPending ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; + } + } + } + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; } +#endif - /* Now manipulate the actual database free-list structure. There are two - ** possibilities. If the free-list is currently empty, or if the first - ** trunk page in the free-list is full, then this page will become a - ** new free-list trunk page. Otherwise, it will become a leaf of the - ** first trunk page in the current free-list. This block tests if it - ** is possible to add the page as a new free-list leaf. - */ - if( nFree!=0 ){ - u32 nLeaf; /* Initial number of leaf cells on trunk page */ + /* Any read-only or read-write transaction implies a read-lock on + ** page 1. So if some other shared-cache client already has a write-lock + ** on page 1, the transaction cannot be opened. */ + rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); + if( SQLITE_OK!=rc ) goto trans_begun; - iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + pBt->initiallyEmpty = (u8)(pBt->nPage==0); + do { + /* Call lockBtree() until either pBt->pPage1 is populated or + ** lockBtree() returns something other than SQLITE_OK. lockBtree() + ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after + ** reading page 1 it discovers that the page-size of the database + ** file is not pBt->pageSize. In this case lockBtree() will update + ** pBt->pageSize to the page-size of the file on disk. + */ + while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); + + if( rc==SQLITE_OK && wrflag ){ + if( pBt->readOnly ){ + rc = SQLITE_READONLY; + }else{ + rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = newDatabase(pBt); + } + } + } + if( rc!=SQLITE_OK ){ - goto freepage_out; + unlockBtreeIfUnused(pBt); } + }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && + btreeInvokeBusyHandler(pBt) ); - nLeaf = get4byte(&pTrunk->aData[4]); - assert( pBt->usableSize>32 ); - if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ - rc = SQLITE_CORRUPT_BKPT; - goto freepage_out; + if( rc==SQLITE_OK ){ + if( p->inTrans==TRANS_NONE ){ + pBt->nTransaction++; +#ifndef SQLITE_OMIT_SHARED_CACHE + if( p->sharable ){ + assert( p->lock.pBtree==p && p->lock.iTable==1 ); + p->lock.eLock = READ_LOCK; + p->lock.pNext = pBt->pLock; + pBt->pLock = &p->lock; + } +#endif } - if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ - /* In this case there is room on the trunk page to insert the page - ** being freed as a new leaf. - ** - ** Note that the trunk page is not really full until it contains - ** usableSize/4 - 2 entries, not usableSize/4 - 8 entries as we have - ** coded. But due to a coding error in versions of SQLite prior to - ** 3.6.0, databases with freelist trunk pages holding more than - ** usableSize/4 - 8 entries will be reported as corrupt. In order - ** to maintain backwards compatibility with older versions of SQLite, - ** we will continue to restrict the number of entries to usableSize/4 - 8 - ** for now. At some point in the future (once everyone has upgraded - ** to 3.6.0 or later) we should consider fixing the conditional above - ** to read "usableSize/4-2" instead of "usableSize/4-8". + p->inTrans = (wrflag?TRANS_WRITE:TRANS_READ); + if( p->inTrans>pBt->inTransaction ){ + pBt->inTransaction = p->inTrans; + } + if( wrflag ){ + MemPage *pPage1 = pBt->pPage1; +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( !pBt->pWriter ); + pBt->pWriter = p; + pBt->isExclusive = (u8)(wrflag>1); +#endif + + /* If the db-size header field is incorrect (as it may be if an old + ** client has been writing the database file), update it now. Doing + ** this sooner rather than later means the database size can safely + ** re-read the database size from page 1 if a savepoint or transaction + ** rollback occurs within the transaction. */ - rc = sqlite3PagerWrite(pTrunk->pDbPage); - if( rc==SQLITE_OK ){ - put4byte(&pTrunk->aData[4], nLeaf+1); - put4byte(&pTrunk->aData[8+nLeaf*4], iPage); - if( pPage && !pBt->secureDelete ){ - sqlite3PagerDontWrite(pPage->pDbPage); + if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pPage1->aData[28], pBt->nPage); } - rc = btreeSetHasContent(pBt, iPage); } - TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); - goto freepage_out; } } - /* If control flows to this point, then it was not possible to add the - ** the page being freed as a leaf page of the first trunk in the free-list. - ** Possibly because the free-list is empty, or possibly because the - ** first trunk in the free-list is full. Either way, the page being freed - ** will become the new first trunk page in the free-list. - */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ - goto freepage_out; - } - rc = sqlite3PagerWrite(pPage->pDbPage); - if( rc!=SQLITE_OK ){ - goto freepage_out; - } - put4byte(pPage->aData, iTrunk); - put4byte(&pPage->aData[4], 0); - put4byte(&pPage1->aData[32], iPage); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); -freepage_out: - if( pPage ){ - pPage->isInit = 0; +trans_begun: + if( rc==SQLITE_OK && wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); } - releasePage(pPage); - releasePage(pTrunk); + + btreeIntegrity(p); + sqlite3BtreeLeave(p); return rc; } -static void freePage(MemPage *pPage, int *pRC){ - if( (*pRC)==SQLITE_OK ){ - *pRC = freePage2(pPage->pBt, pPage, pPage->pgno); - } -} + +#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Free any overflow pages associated with the given Cell. +** Set the pointer-map entries for all children of page pPage. Also, if +** pPage contains cells that point to overflow pages, set the pointer +** map entries for the overflow pages as well. */ -static int clearCell(MemPage *pPage, unsigned char *pCell){ +static int setChildPtrmaps(MemPage *pPage){ + int i; /* Counter variable */ + int nCell; /* Number of cells in page pPage */ + int rc; /* Return code */ BtShared *pBt = pPage->pBt; - CellInfo info; - Pgno ovflPgno; - int rc; - int nOvfl; - u32 ovflPageSize; + u8 isInitOrig = pPage->isInit; + Pgno pgno = pPage->pgno; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - btreeParseCellPtr(pPage, pCell, &info); - if( info.iOverflow==0 ){ - return SQLITE_OK; /* No overflow pages. Return without doing anything */ + rc = btreeInitPage(pPage); + if( rc!=SQLITE_OK ){ + goto set_child_ptrmaps_out; } - ovflPgno = get4byte(&pCell[info.iOverflow]); - assert( pBt->usableSize > 4 ); - ovflPageSize = pBt->usableSize - 4; - nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; - assert( ovflPgno==0 || nOvfl>0 ); - while( nOvfl-- ){ - Pgno iNext = 0; - MemPage *pOvfl = 0; - if( ovflPgno<2 || ovflPgno>btreePagecount(pBt) ){ - /* 0 is not a legal page number and page 1 cannot be an - ** overflow page. Therefore if ovflPgno<2 or past the end of the - ** file the database must be corrupt. */ - return SQLITE_CORRUPT_BKPT; - } - if( nOvfl ){ - rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); - if( rc ) return rc; - } + nCell = pPage->nCell; - if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) - && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 - ){ - /* There is no reason any cursor should have an outstanding reference - ** to an overflow page belonging to a cell that is being deleted/updated. - ** So if there exists more than one reference to this page, then it - ** must not really be an overflow page and the database must be corrupt. - ** It is helpful to detect this before calling freePage2(), as - ** freePage2() may zero the page contents if secure-delete mode is - ** enabled. If this 'overflow' page happens to be a page that the - ** caller is iterating through or using in some other way, this - ** can be problematic. - */ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = freePage2(pBt, pOvfl, ovflPgno); - } + for(i=0; ipDbPage); + ptrmapPutOvflPtr(pPage, pCell, &rc); + + if( !pPage->leaf ){ + Pgno childPgno = get4byte(pCell); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); } - if( rc ) return rc; - ovflPgno = iNext; } - return SQLITE_OK; + + if( !pPage->leaf ){ + Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + +set_child_ptrmaps_out: + pPage->isInit = isInitOrig; + return rc; } /* -** Create the byte sequence used to represent a cell on page pPage -** and write that byte sequence into pCell[]. Overflow pages are -** allocated and filled in as necessary. The calling procedure -** is responsible for making sure sufficient space has been allocated -** for pCell[]. +** Somewhere on pPage is a pointer to page iFrom. Modify this pointer so +** that it points to iTo. Parameter eType describes the type of pointer to +** be modified, as follows: ** -** Note that pCell does not necessary need to point to the pPage->aData -** area. pCell might point to some temporary storage. The cell will -** be constructed in this temporary area then copied into pPage->aData -** later. +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** page of pPage. +** +** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow +** page pointed to by one of the cells on pPage. +** +** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next +** overflow page in the list. */ -static int fillInCell( - MemPage *pPage, /* The page that contains the cell */ - unsigned char *pCell, /* Complete text of the cell */ - const void *pKey, i64 nKey, /* The key */ - const void *pData,int nData, /* The data */ - int nZero, /* Extra zero bytes to append to pData */ - int *pnSize /* Write cell size here */ -){ - int nPayload; - const u8 *pSrc; - int nSrc, n, rc; - int spaceLeft; - MemPage *pOvfl = 0; - MemPage *pToRelease = 0; - unsigned char *pPrior; - unsigned char *pPayload; - BtShared *pBt = pPage->pBt; - Pgno pgnoOvfl = 0; - int nHeader; - CellInfo info; - +static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - - /* pPage is not necessarily writeable since pCell might be auxiliary - ** buffer space that is separate from the pPage buffer area */ - assert( pCellaData || pCell>=&pPage->aData[pBt->pageSize] - || sqlite3PagerIswriteable(pPage->pDbPage) ); - - /* Fill in the header. */ - nHeader = 0; - if( !pPage->leaf ){ - nHeader += 4; - } - if( pPage->hasData ){ - nHeader += putVarint(&pCell[nHeader], nData+nZero); - }else{ - nData = nZero = 0; - } - nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - btreeParseCellPtr(pPage, pCell, &info); - assert( info.nHeader==nHeader ); - assert( info.nKey==nKey ); - assert( info.nData==(u32)(nData+nZero) ); - - /* Fill in the payload */ - nPayload = nData + nZero; - if( pPage->intKey ){ - pSrc = pData; - nSrc = nData; - nData = 0; - }else{ - if( NEVER(nKey>0x7fffffff || pKey==0) ){ + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( eType==PTRMAP_OVERFLOW2 ){ + /* The pointer is always the first 4 bytes of the page in this case. */ + if( get4byte(pPage->aData)!=iFrom ){ return SQLITE_CORRUPT_BKPT; } - nPayload += (int)nKey; - pSrc = pKey; - nSrc = (int)nKey; - } - *pnSize = info.nSize; - spaceLeft = info.nLocal; - pPayload = &pCell[nHeader]; - pPrior = &pCell[info.iOverflow]; + put4byte(pPage->aData, iTo); + }else{ + u8 isInitOrig = pPage->isInit; + int i; + int nCell; - while( nPayload>0 ){ - if( spaceLeft==0 ){ -#ifndef SQLITE_OMIT_AUTOVACUUM - Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ - if( pBt->autoVacuum ){ - do{ - pgnoOvfl++; - } while( - PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) - ); - } -#endif - rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); -#ifndef SQLITE_OMIT_AUTOVACUUM - /* If the database supports auto-vacuum, and the second or subsequent - ** overflow page is being allocated, add an entry to the pointer-map - ** for that page now. - ** - ** If this is the first overflow page, then write a partial entry - ** to the pointer-map. If we write nothing to this pointer-map slot, - ** then the optimistic overflow chain processing in clearCell() - ** may misinterpret the uninitialised values and delete the - ** wrong pages from the database. - */ - if( pBt->autoVacuum && rc==SQLITE_OK ){ - u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); - ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); - if( rc ){ - releasePage(pOvfl); + btreeInitPage(pPage); + nCell = pPage->nCell; + + for(i=0; iaData[pPage->hdrOffset+8])!=iFrom ){ + return SQLITE_CORRUPT_BKPT; } - - /* If pToRelease is not zero than pPrior points into the data area - ** of pToRelease. Make sure pToRelease is still writeable. */ - assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); - - /* If pPrior is part of the data area of pPage, then make sure pPage - ** is still writeable */ - assert( pPrioraData || pPrior>=&pPage->aData[pBt->pageSize] - || sqlite3PagerIswriteable(pPage->pDbPage) ); - - put4byte(pPrior, pgnoOvfl); - releasePage(pToRelease); - pToRelease = pOvfl; - pPrior = pOvfl->aData; - put4byte(pPrior, 0); - pPayload = &pOvfl->aData[4]; - spaceLeft = pBt->usableSize - 4; + put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); } - n = nPayload; - if( n>spaceLeft ) n = spaceLeft; - - /* If pToRelease is not zero than pPayload points into the data area - ** of pToRelease. Make sure pToRelease is still writeable. */ - assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); - - /* If pPayload is part of the data area of pPage, then make sure pPage - ** is still writeable */ - assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] - || sqlite3PagerIswriteable(pPage->pDbPage) ); - if( nSrc>0 ){ - if( n>nSrc ) n = nSrc; - assert( pSrc ); - memcpy(pPayload, pSrc, n); - }else{ - memset(pPayload, 0, n); - } - nPayload -= n; - pPayload += n; - pSrc += n; - nSrc -= n; - spaceLeft -= n; - if( nSrc==0 ){ - nSrc = nData; - pSrc = pData; - } + pPage->isInit = isInitOrig; } - releasePage(pToRelease); return SQLITE_OK; } + /* -** Remove the i-th cell from pPage. This routine effects pPage only. -** The cell content is not freed or deallocated. It is assumed that -** the cell content has been copied someplace else. This routine just -** removes the reference to the cell from pPage. +** Move the open database page pDbPage to location iFreePage in the +** database. The pDbPage reference remains valid. ** -** "sz" must be the number of bytes in the cell. +** The isCommit flag indicates that there is no need to remember that +** the journal needs to be sync()ed before database page pDbPage->pgno +** can be written to. The caller has already promised not to write to that +** page. */ -static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ - int i; /* Loop counter */ - u32 pc; /* Offset to cell content of cell being deleted */ - u8 *data; /* pPage->aData */ - u8 *ptr; /* Used to move bytes around within data[] */ - int rc; /* The return code */ - int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ +static int relocatePage( + BtShared *pBt, /* Btree */ + MemPage *pDbPage, /* Open page to move */ + u8 eType, /* Pointer map 'type' entry for pDbPage */ + Pgno iPtrPage, /* Pointer map 'page-no' entry for pDbPage */ + Pgno iFreePage, /* The location to move pDbPage to */ + int isCommit /* isCommit flag passed to sqlite3PagerMovepage */ +){ + MemPage *pPtrPage; /* The page that contains a pointer to pDbPage */ + Pgno iDbPage = pDbPage->pgno; + Pager *pPager = pBt->pPager; + int rc; - if( *pRC ) return; + assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || + eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pDbPage->pBt==pBt ); - assert( idx>=0 && idxnCell ); - assert( sz==cellSize(pPage, idx) ); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - data = pPage->aData; - ptr = &data[pPage->cellOffset + 2*idx]; - pc = get2byte(ptr); - hdr = pPage->hdrOffset; - testcase( pc==get2byte(&data[hdr+5]) ); - testcase( pc+sz==pPage->pBt->usableSize ); - if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; + /* Move page iDbPage from its current location to page number iFreePage */ + TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + iDbPage, iFreePage, iPtrPage, eType)); + rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); + if( rc!=SQLITE_OK ){ + return rc; } - rc = freeSpace(pPage, pc, sz); - if( rc ){ - *pRC = rc; - return; + pDbPage->pgno = iFreePage; + + /* If pDbPage was a btree-page, then it may have child pages and/or cells + ** that point to overflow pages. The pointer map entries for all these + ** pages need to be changed. + ** + ** If pDbPage is an overflow page, then the first 4 bytes may store a + ** pointer to a subsequent overflow page. If this is the case, then + ** the pointer map needs to be updated for the subsequent overflow page. + */ + if( eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ){ + rc = setChildPtrmaps(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + Pgno nextOvfl = get4byte(pDbPage->aData); + if( nextOvfl!=0 ){ + ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage, &rc); + if( rc!=SQLITE_OK ){ + return rc; + } + } } - for(i=idx+1; inCell; i++, ptr+=2){ - ptr[0] = ptr[2]; - ptr[1] = ptr[3]; + + /* Fix the database pointer on page iPtrPage that pointed at iDbPage so + ** that it points at iFreePage. Also fix the pointer map entry for + ** iPtrPage. + */ + if( eType!=PTRMAP_ROOTPAGE ){ + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pPtrPage->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pPtrPage); + return rc; + } + rc = modifyPagePointer(pPtrPage, iDbPage, iFreePage, eType); + releasePage(pPtrPage); + if( rc==SQLITE_OK ){ + ptrmapPut(pBt, iFreePage, eType, iPtrPage, &rc); + } } - pPage->nCell--; - put2byte(&data[hdr+3], pPage->nCell); - pPage->nFree += 2; + return rc; } +/* Forward declaration required by incrVacuumStep(). */ +static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); + /* -** Insert a new cell on pPage at cell index "i". pCell points to the -** content of the cell. +** Perform a single step of an incremental-vacuum. If successful, +** return SQLITE_OK. If there is no work to do (and therefore no +** point in calling this function again), return SQLITE_DONE. ** -** If the cell content will fit on the page, then put it there. If it -** will not fit, then make a copy of the cell content into pTemp if -** pTemp is not null. Regardless of pTemp, allocate a new entry -** in pPage->aOvfl[] and make it point to the cell content (either -** in pTemp or the original pCell) and also record its index. -** Allocating a new entry in pPage->aCell[] implies that -** pPage->nOverflow is incremented. +** More specificly, this function attempts to re-organize the +** database so that the last page of the file currently in use +** is no longer in use. ** -** If nSkip is non-zero, then do not copy the first nSkip bytes of the -** cell. The caller will overwrite them after this function returns. If -** nSkip is non-zero, then pCell may not point to an invalid memory location -** (but pCell+nSkip is always valid). +** If the nFin parameter is non-zero, this function assumes +** that the caller will keep calling incrVacuumStep() until +** it returns SQLITE_DONE or an error, and that nFin is the +** number of pages the database file will contain after this +** process is complete. If nFin is zero, it is assumed that +** incrVacuumStep() will be called a finite amount of times +** which may or may not empty the freelist. A full autovacuum +** has nFin>0. A "PRAGMA incremental_vacuum" has nFin==0. */ -static void insertCell( - MemPage *pPage, /* Page into which we are copying */ - int i, /* New cell becomes the i-th cell of the page */ - u8 *pCell, /* Content of the new cell */ - int sz, /* Bytes of content in pCell */ - u8 *pTemp, /* Temp storage space for pCell, if needed */ - Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ - int *pRC /* Read and write return code from here */ -){ - int idx = 0; /* Where to write new cell content in data[] */ - int j; /* Loop counter */ - int end; /* First byte past the last cell pointer in data[] */ - int ins; /* Index in data[] where new cell pointer is inserted */ - int cellOffset; /* Address of first cell pointer in data[] */ - u8 *data; /* The content of the whole page */ - u8 *ptr; /* Used for moving information around in data[] */ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ + Pgno nFreeList; /* Number of pages still on the free-list */ + int rc; - int nSkip = (iChild ? 4 : 0); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( iLastPg>nFin ); - if( *pRC ) return; + if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ + u8 eType; + Pgno iPtrPage; - assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); - assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); - assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - /* The cell should normally be sized correctly. However, when moving a - ** malformed cell from a leaf page to an interior page, if the cell size - ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size - ** might be less than 8 (leaf-size + pointer) on the interior node. Hence - ** the term after the || in the following assert(). */ - assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); - if( pPage->nOverflow || sz+2>pPage->nFree ){ - if( pTemp ){ - memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); - pCell = pTemp; - } - if( iChild ){ - put4byte(pCell, iChild); + nFreeList = get4byte(&pBt->pPage1->aData[36]); + if( nFreeList==0 ){ + return SQLITE_DONE; } - j = pPage->nOverflow++; - assert( j<(int)(sizeof(pPage->aOvfl)/sizeof(pPage->aOvfl[0])) ); - pPage->aOvfl[j].pCell = pCell; - pPage->aOvfl[j].idx = (u16)i; - }else{ - int rc = sqlite3PagerWrite(pPage->pDbPage); + + rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage); if( rc!=SQLITE_OK ){ - *pRC = rc; - return; - } - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - data = pPage->aData; - cellOffset = pPage->cellOffset; - end = cellOffset + 2*pPage->nCell; - ins = cellOffset + 2*i; - rc = allocateSpace(pPage, sz, &idx); - if( rc ){ *pRC = rc; return; } - /* The allocateSpace() routine guarantees the following two properties - ** if it returns success */ - assert( idx >= end+2 ); - assert( idx+sz <= pPage->pBt->usableSize ); - pPage->nCell++; - pPage->nFree -= (u16)(2 + sz); - memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); - if( iChild ){ - put4byte(&data[idx], iChild); - } - for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){ - ptr[0] = ptr[-2]; - ptr[1] = ptr[-1]; + return rc; } - put2byte(&data[ins], idx); - put2byte(&data[pPage->hdrOffset+3], pPage->nCell); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pPage->pBt->autoVacuum ){ - /* The cell may contain a pointer to an overflow page. If so, write - ** the entry for the overflow page into the pointer map. - */ - ptrmapPutOvflPtr(pPage, pCell, pRC); + if( eType==PTRMAP_ROOTPAGE ){ + return SQLITE_CORRUPT_BKPT; } -#endif - } -} - -/* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. -*/ -static void assemblePage( - MemPage *pPage, /* The page to be assemblied */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ -){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( eType==PTRMAP_FREEPAGE ){ + if( nFin==0 ){ + /* Remove the page from the files free-list. This is not required + ** if nFin is non-zero. In that case, the free-list will be + ** truncated to zero after this function returns, so it doesn't + ** matter if it still contains some garbage entries. + */ + Pgno iFreePg; + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( iFreePg==iLastPg ); + releasePage(pFreePg); + } + } else { + Pgno iFreePg; /* Index of free page to move pLastPg to */ + MemPage *pLastPg; - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + if( rc!=SQLITE_OK ){ + return rc; + } - pCellptr = &data[pPage->cellOffset + nCell*2]; - cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ - pCellptr -= 2; - cellbody -= aSize[i]; - put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], aSize[i]); + /* If nFin is zero, this loop runs exactly once and page pLastPg + ** is swapped with the first free page pulled off the free list. + ** + ** On the other hand, if nFin is greater than zero, then keep + ** looping until a free-page located within the first nFin pages + ** of the file is found. + */ + do { + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); + if( rc!=SQLITE_OK ){ + releasePage(pLastPg); + return rc; + } + releasePage(pFreePg); + }while( nFin!=0 && iFreePg>nFin ); + assert( iFreePgpDbPage); + if( rc==SQLITE_OK ){ + rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); + } + releasePage(pLastPg); + if( rc!=SQLITE_OK ){ + return rc; + } + } } - put2byte(&data[hdr+3], nCell); - put2byte(&data[hdr+5], cellbody); - pPage->nFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; + + if( nFin==0 ){ + iLastPg--; + while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ + if( PTRMAP_ISPAGE(pBt, iLastPg) ){ + MemPage *pPg; + rc = btreeGetPage(pBt, iLastPg, &pPg, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pPg->pDbPage); + releasePage(pPg); + if( rc!=SQLITE_OK ){ + return rc; + } + } + iLastPg--; + } + sqlite3PagerTruncateImage(pBt->pPager, iLastPg); + pBt->nPage = iLastPg; + } + return SQLITE_OK; } /* -** The following parameters determine how many adjacent pages get involved -** in a balancing operation. NN is the number of neighbors on either side -** of the page that participate in the balancing operation. NB is the -** total number of pages that participate, including the target page and -** NN neighbors on either side. +** A write-transaction must be opened before calling this function. +** It performs a single unit of work towards an incremental vacuum. ** -** The minimum value of NN is 1 (of course). Increasing NN above 1 -** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance -** in exchange for a larger degradation in INSERT and UPDATE performance. -** The value of NN appears to give the best results overall. +** If the incremental vacuum is finished after this function has run, +** SQLITE_DONE is returned. If it is not finished, but no error occurred, +** SQLITE_OK is returned. Otherwise an SQLite error code. */ -#define NN 1 /* Number of neighbors on either side of pPage */ -#define NB (NN*2+1) /* Total pages involved in the balance */ +SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); + if( !pBt->autoVacuum ){ + rc = SQLITE_DONE; + }else{ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + } + sqlite3BtreeLeave(p); + return rc; +} -#ifndef SQLITE_OMIT_QUICKBALANCE /* -** This version of balance() handles the common special case where -** a new entry is being inserted on the extreme right-end of the -** tree, in other words, when the new entry will become the largest -** entry in the tree. -** -** Instead of trying to balance the 3 right-most leaf pages, just add -** a new page to the right-hand side and put the one new entry in -** that page. This leaves the right side of the tree somewhat -** unbalanced. But odds are that we will be inserting new entries -** at the end soon afterwards so the nearly empty page will quickly -** fill up. On average. -** -** pPage is the leaf page which is the right-most page in the tree. -** pParent is its parent. pPage must have a single overflow entry -** which is also the right-most entry on the page. +** This routine is called prior to sqlite3PagerCommit when a transaction +** is commited for an auto-vacuum database. ** -** The pSpace buffer is used to store a temporary copy of the divider -** cell that will be inserted into pParent. Such a cell consists of a 4 -** byte page number followed by a variable length integer. In other -** words, at most 13 bytes. Hence the pSpace buffer must be at -** least 13 bytes in size. +** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages +** the database file should be truncated to during the commit process. +** i.e. the database has been reorganized so that only the first *pnTrunc +** pages are in use. */ -static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ - BtShared *const pBt = pPage->pBt; /* B-Tree Database */ - MemPage *pNew; /* Newly allocated page */ - int rc; /* Return Code */ - Pgno pgnoNew; /* Page number of pNew */ +static int autoVacuumCommit(BtShared *pBt){ + int rc = SQLITE_OK; + Pager *pPager = pBt->pPager; + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - assert( pPage->nOverflow==1 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + invalidateAllOverflowCache(pBt); + assert(pBt->autoVacuum); + if( !pBt->incrVacuum ){ + Pgno nFin; /* Number of pages in database after autovacuuming */ + Pgno nFree; /* Number of pages on the freelist initially */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno iFree; /* The next page to be freed */ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nOrig; /* Database size before freeing */ - /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; + nOrig = btreePagecount(pBt); + if( PTRMAP_ISPAGE(pBt, nOrig) || nOrig==PENDING_BYTE_PAGE(pBt) ){ + /* It is not possible to create a database for which the final page + ** is either a pointer-map page or the pending-byte page. If one + ** is encountered, this indicates corruption. + */ + return SQLITE_CORRUPT_BKPT; + } - /* Allocate a new page. This page will become the right-sibling of - ** pPage. Make the parent page writable, so that the new divider cell - ** may be inserted. If both these operations are successful, proceed. - */ - rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + nFree = get4byte(&pBt->pPage1->aData[36]); + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinnOrig ) return SQLITE_CORRUPT_BKPT; - if( rc==SQLITE_OK ){ + for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ + rc = incrVacuumStep(pBt, nFin, iFree); + } + if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + put4byte(&pBt->pPage1->aData[28], nFin); + sqlite3PagerTruncateImage(pBt->pPager, nFin); + pBt->nPage = nFin; + } + if( rc!=SQLITE_OK ){ + sqlite3PagerRollback(pPager); + } + } - u8 *pOut = &pSpace[4]; - u8 *pCell = pPage->aOvfl[0].pCell; - u16 szCell = cellSizePtr(pPage, pCell); - u8 *pStop; + assert( nRef==sqlite3PagerRefcount(pPager) ); + return rc; +} - assert( sqlite3PagerIswriteable(pNew->pDbPage) ); - assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); - zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); +#else /* ifndef SQLITE_OMIT_AUTOVACUUM */ +# define setChildPtrmaps(x) SQLITE_OK +#endif - /* If this is an auto-vacuum database, update the pointer map - ** with entries for the new page, and any pointer from the - ** cell on the page to an overflow page. If either of these - ** operations fails, the return code is set, but the contents - ** of the parent page are still manipulated by thh code below. - ** That is Ok, at this point the parent page is guaranteed to - ** be marked as dirty. Returning an error code will cause a - ** rollback, undoing any changes made to the parent page. - */ - if( ISAUTOVACUUM ){ - ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); - if( szCell>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); +/* +** This routine does the first phase of a two-phase commit. This routine +** causes a rollback journal to be created (if it does not already exist) +** and populated with enough information so that if a power loss occurs +** the database can be restored to its original state by playing back +** the journal. Then the contents of the journal are flushed out to +** the disk. After the journal is safely on oxide, the changes to the +** database are written into the database file and flushed to oxide. +** At the end of this call, the rollback journal still exists on the +** disk and we are still holding all locks, so the transaction has not +** committed. See sqlite3BtreeCommitPhaseTwo() for the second phase of the +** commit process. +** +** This call is a no-op if no write-transaction is currently active on pBt. +** +** Otherwise, sync the database file for the btree pBt. zMaster points to +** the name of a master journal file that should be written into the +** individual journal file, or is NULL, indicating no master journal file +** (single database transaction). +** +** When this is called, the master journal should already have been +** created, populated with this journal pointer and synced to disk. +** +** Once this is routine has returned, the only thing required to commit +** the write-transaction for this database file is to delete the journal. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ + int rc = SQLITE_OK; + if( p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + rc = autoVacuumCommit(pBt); + if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); + return rc; } } - - /* Create a divider cell to insert into pParent. The divider cell - ** consists of a 4-byte page number (the page number of pPage) and - ** a variable length key value (which must be the same value as the - ** largest key on pPage). - ** - ** To find the largest key value on pPage, first find the right-most - ** cell on pPage. The first two fields of this cell are the - ** record-length (a variable length integer at most 32-bits in size) - ** and the key value (a variable length integer, may have any value). - ** The first of the while(...) loops below skips over the record-length - ** field. The second while(...) loop copies the key value from the - ** cell on pPage into the pSpace buffer. - */ - pCell = findCell(pPage, pPage->nCell-1); - pStop = &pCell[9]; - while( (*(pCell++)&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); - - /* Set the right-child pointer of pParent to point to the new page. */ - put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); - - /* Release the reference to the new page. */ - releasePage(pNew); +#endif + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); + sqlite3BtreeLeave(p); } - return rc; } -#endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 0 /* -** This function does not contribute anything to the operation of SQLite. -** it is sometimes activated temporarily while debugging code responsible -** for setting pointer-map entries. +** This function is called from both BtreeCommitPhaseTwo() and BtreeRollback() +** at the conclusion of a transaction. */ -static int ptrmapCheckPages(MemPage **apPage, int nPage){ - int i, j; - for(i=0; ipBt; - assert( pPage->isInit ); +static void btreeEndTransaction(Btree *p){ + BtShared *pBt = p->pBt; + assert( sqlite3BtreeHoldsMutex(p) ); - for(j=0; jnCell; j++){ - CellInfo info; - u8 *z; - - z = findCell(pPage, j); - btreeParseCellPtr(pPage, z, &info); - if( info.iOverflow ){ - Pgno ovfl = get4byte(&z[info.iOverflow]); - ptrmapGet(pBt, ovfl, &e, &n); - assert( n==pPage->pgno && e==PTRMAP_OVERFLOW1 ); - } - if( !pPage->leaf ){ - Pgno child = get4byte(z); - ptrmapGet(pBt, child, &e, &n); - assert( n==pPage->pgno && e==PTRMAP_BTREE ); + btreeClearHasContent(pBt); + if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){ + /* If there are other active statements that belong to this database + ** handle, downgrade to a read-only transaction. The other statements + ** may still be reading from the database. */ + downgradeAllSharedCacheTableLocks(p); + p->inTrans = TRANS_READ; + }else{ + /* If the handle had any kind of transaction open, decrement the + ** transaction count of the shared btree. If the transaction count + ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() + ** call below will unlock the pager. */ + if( p->inTrans!=TRANS_NONE ){ + clearAllSharedCacheTableLocks(p); + pBt->nTransaction--; + if( 0==pBt->nTransaction ){ + pBt->inTransaction = TRANS_NONE; } } - if( !pPage->leaf ){ - Pgno child = get4byte(&pPage->aData[pPage->hdrOffset+8]); - ptrmapGet(pBt, child, &e, &n); - assert( n==pPage->pgno && e==PTRMAP_BTREE ); - } + + /* Set the current transaction state to TRANS_NONE and unlock the + ** pager if this call closed the only read or write transaction. */ + p->inTrans = TRANS_NONE; + unlockBtreeIfUnused(pBt); } - return 1; + + btreeIntegrity(p); } -#endif /* -** This function is used to copy the contents of the b-tree node stored -** on page pFrom to page pTo. If page pFrom was not a leaf page, then -** the pointer-map entries for each child page are updated so that the -** parent page stored in the pointer map is page pTo. If pFrom contained -** any cells with overflow page pointers, then the corresponding pointer -** map entries are also updated so that the parent page is page pTo. +** Commit the transaction currently in progress. ** -** If pFrom is currently carrying any overflow cells (entries in the -** MemPage.aOvfl[] array), they are not copied to pTo. +** This routine implements the second phase of a 2-phase commit. The +** sqlite3BtreeCommitPhaseOne() routine does the first phase and should +** be invoked prior to calling this routine. The sqlite3BtreeCommitPhaseOne() +** routine did all the work of writing information out to disk and flushing the +** contents so that they are written onto the disk platter. All this +** routine has to do is delete or truncate or zero the header in the +** the rollback journal (which causes the transaction to commit) and +** drop locks. ** -** Before returning, page pTo is reinitialized using btreeInitPage(). +** Normally, if an error occurs while the pager layer is attempting to +** finalize the underlying journal file, this function returns an error and +** the upper layer will attempt a rollback. However, if the second argument +** is non-zero then this b-tree transaction is part of a multi-file +** transaction. In this case, the transaction has already been committed +** (by deleting a master journal file) and the caller will ignore this +** functions return code. So, even if an error occurs in the pager layer, +** reset the b-tree objects internal state to indicate that the write +** transaction has been closed. This is quite safe, as the pager will have +** transitioned to the error state. ** -** The performance of this function is not critical. It is only used by -** the balance_shallower() and balance_deeper() procedures, neither of -** which are called often under normal circumstances. +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. */ -static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ - if( (*pRC)==SQLITE_OK ){ - BtShared * const pBt = pFrom->pBt; - u8 * const aFrom = pFrom->aData; - u8 * const aTo = pTo->aData; - int const iFromHdr = pFrom->hdrOffset; - int const iToHdr = ((pTo->pgno==1) ? 100 : 0); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ + + if( p->inTrans==TRANS_NONE ) return SQLITE_OK; + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the handle has a write-transaction open, commit the shared-btrees + ** transaction and set the shared state to TRANS_READ. + */ + if( p->inTrans==TRANS_WRITE ){ int rc; - int iData; - - - assert( pFrom->isInit ); - assert( pFrom->nFree>=iToHdr ); - assert( get2byte(&aFrom[iFromHdr+5])<=pBt->usableSize ); - - /* Copy the b-tree node content from page pFrom to page pTo. */ - iData = get2byte(&aFrom[iFromHdr+5]); - memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); - memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); - - /* Reinitialize page pTo so that the contents of the MemPage structure - ** match the new data. The initialization of pTo can actually fail under - ** fairly obscure circumstances, even though it is a copy of initialized - ** page pFrom. - */ - pTo->isInit = 0; - rc = btreeInitPage(pTo); - if( rc!=SQLITE_OK ){ - *pRC = rc; - return; - } - - /* If this is an auto-vacuum database, update the pointer-map entries - ** for any b-tree or overflow pages that pTo now contains the pointers to. - */ - if( ISAUTOVACUUM ){ - *pRC = setChildPtrmaps(pTo); + BtShared *pBt = p->pBt; + assert( pBt->inTransaction==TRANS_WRITE ); + assert( pBt->nTransaction>0 ); + rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); + if( rc!=SQLITE_OK && bCleanup==0 ){ + sqlite3BtreeLeave(p); + return rc; } + pBt->inTransaction = TRANS_READ; } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return SQLITE_OK; } /* -** This routine redistributes cells on the iParentIdx'th child of pParent -** (hereafter "the page") and up to 2 siblings so that all pages have about the -** same amount of free space. Usually a single sibling on either side of the -** page are used in the balancing, though both siblings might come from one -** side if the page is the first or last child of its parent. If the page -** has fewer than 2 siblings (something which can only happen if the page -** is a root page or a child of a root page) then all available siblings -** participate in the balancing. -** -** The number of siblings of the page might be increased or decreased by -** one or two in an effort to keep pages nearly full but not over full. -** -** Note that when this routine is called, some of the cells on the page -** might not actually be stored in MemPage.aData[]. This can happen -** if the page is overfull. This routine ensures that all cells allocated -** to the page and its siblings fit into MemPage.aData[] before returning. -** -** In the course of balancing the page and its siblings, cells may be -** inserted into or removed from the parent page (pParent). Doing so -** may cause the parent page to become overfull or underfull. If this -** happens, it is the responsibility of the caller to invoke the correct -** balancing routine to fix this problem (see the balance() routine). +** Do both phases of a commit. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + rc = sqlite3BtreeCommitPhaseOne(p, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeCommitPhaseTwo(p, 0); + } + sqlite3BtreeLeave(p); + return rc; +} + +#ifndef NDEBUG +/* +** Return the number of write-cursors open on this handle. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. ** -** If this routine fails for any reason, it might leave the database -** in a corrupted state. So if this routine fails, the database should -** be rolled back. +** For the purposes of this routine, a write-cursor is any cursor that +** is capable of writing to the databse. That means the cursor was +** originally opened for writing and the cursor has not be disabled +** by having its state changed to CURSOR_FAULT. +*/ +static int countWriteCursors(BtShared *pBt){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + +/* +** This routine sets the state to CURSOR_FAULT and the error +** code to errCode for every cursor on BtShared that pBtree +** references. ** -** The third argument to this function, aOvflSpace, is a pointer to a -** buffer big enough to hold one page. If while inserting cells into the parent -** page (pParent) the parent page becomes overfull, this buffer is -** used to store the parent's overflow cells. Because this function inserts -** a maximum of four divider cells into the parent page, and the maximum -** size of a cell stored within an internal node is always less than 1/4 -** of the page-size, the aOvflSpace[] buffer is guaranteed to be large -** enough for all overflow cells. +** Every cursor is tripped, including cursors that belong +** to other database connections that happen to be sharing +** the cache with pBtree. ** -** If aOvflSpace is set to a null pointer, this function returns -** SQLITE_NOMEM. +** This routine gets called when a rollback occurs. +** All cursors using the same cache must be tripped +** to prevent them from trying to use the btree after +** the rollback. The rollback may have deleted tables +** or moved root pages, so it is not sufficient to +** save the state of the cursor. The cursor must be +** invalidated. */ -static int balance_nonroot( - MemPage *pParent, /* Parent page of siblings being balanced */ - int iParentIdx, /* Index of "the page" in pParent */ - u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ - int isRoot /* True if pParent is a root-page */ -){ - BtShared *pBt; /* The whole database */ - int nCell = 0; /* Number of cells in apCell[] */ - int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ - int nNew = 0; /* Number of pages in apNew[] */ - int nOld; /* Number of pages in apOld[] */ - int i, j, k; /* Loop counters */ - int nxDiv; /* Next divider slot in pParent->aCell[] */ - int rc = SQLITE_OK; /* The return code */ - u16 leafCorrection; /* 4 if pPage is a leaf. 0 if not */ - int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ - int usableSpace; /* Bytes in pPage beyond the header */ - int pageFlags; /* Value of pPage->aData[0] */ - int subtotal; /* Subtotal of bytes in cells on one page */ - int iSpace1 = 0; /* First unused byte of aSpace1[] */ - int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ - int szScratch; /* Size of scratch memory requested */ - MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ - MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ - u8 *pRight; /* Location in parent of right-sibling pointer */ - u8 *apDiv[NB-1]; /* Divider cells in pParent */ - int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ - u8 **apCell = 0; /* All cells begin balanced */ - u16 *szCell; /* Local size of all cells in apCell[] */ - u8 *aSpace1; /* Space for copies of dividers cells */ - Pgno pgno; /* Temp var to store a page number in */ +SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ + BtCursor *p; + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } + } + sqlite3BtreeLeave(pBtree); +} - pBt = pParent->pBt; - assert( sqlite3_mutex_held(pBt->mutex) ); - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); +/* +** Rollback the transaction in progress. All cursors will be +** invalided by this operation. Any attempt to use a cursor +** that was open at the beginning of this operation will result +** in an error. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p){ + int rc; + BtShared *pBt = p->pBt; + MemPage *pPage1; -#if 0 - TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); + sqlite3BtreeEnter(p); + rc = saveAllCursors(pBt, 0, 0); +#ifndef SQLITE_OMIT_SHARED_CACHE + if( rc!=SQLITE_OK ){ + /* This is a horrible situation. An IO or malloc() error occurred whilst + ** trying to save cursor positions. If this is an automatic rollback (as + ** the result of a constraint, malloc() failure or IO error) then + ** the cache may be internally inconsistent (not contain valid trees) so + ** we cannot simply return the error to the caller. Instead, abort + ** all queries that may be using any of the cursors that failed to save. + */ + sqlite3BtreeTripAllCursors(p, rc); + } #endif + btreeIntegrity(p); - /* At this point pParent may have at most one overflow cell. And if - ** this overflow cell is present, it must be the cell with - ** index iParentIdx. This scenario comes about when this function - ** is called (indirectly) from sqlite3BtreeDelete(). - */ - assert( pParent->nOverflow==0 || pParent->nOverflow==1 ); - assert( pParent->nOverflow==0 || pParent->aOvfl[0].idx==iParentIdx ); - - if( !aOvflSpace ){ - return SQLITE_NOMEM; - } + if( p->inTrans==TRANS_WRITE ){ + int rc2; - /* Find the sibling pages to balance. Also locate the cells in pParent - ** that divide the siblings. An attempt is made to find NN siblings on - ** either side of pPage. More siblings are taken from one side, however, - ** if there are fewer than NN siblings on the other side. If pParent - ** has NB or fewer children then all children of pParent are taken. - ** - ** This loop also drops the divider cells from the parent page. This - ** way, the remainder of the function does not have to deal with any - ** overflow cells in the parent page, since if any existed they will - ** have already been removed. - */ - i = pParent->nOverflow + pParent->nCell; - if( i<2 ){ - nxDiv = 0; - nOld = i+1; - }else{ - nOld = 3; - if( iParentIdx==0 ){ - nxDiv = 0; - }else if( iParentIdx==i ){ - nxDiv = i-2; - }else{ - nxDiv = iParentIdx-1; - } - i = 2; - } - if( (i+nxDiv-pParent->nOverflow)==pParent->nCell ){ - pRight = &pParent->aData[pParent->hdrOffset+8]; - }else{ - pRight = findCell(pParent, i+nxDiv-pParent->nOverflow); - } - pgno = get4byte(pRight); - while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); - if( rc ){ - memset(apOld, 0, (i+1)*sizeof(MemPage*)); - goto balance_cleanup; + assert( TRANS_WRITE==pBt->inTransaction ); + rc2 = sqlite3PagerRollback(pBt->pPager); + if( rc2!=SQLITE_OK ){ + rc = rc2; } - nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; - if( (i--)==0 ) break; - - if( i+nxDiv==pParent->aOvfl[0].idx && pParent->nOverflow ){ - apDiv[i] = pParent->aOvfl[0].pCell; - pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); - pParent->nOverflow = 0; - }else{ - apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); - pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); - /* Drop the cell from the parent page. apDiv[i] still points to - ** the cell within the parent, even though it has been dropped. - ** This is safe because dropping a cell only overwrites the first - ** four bytes of it, and this function does not need the first - ** four bytes of the divider cell. So the pointer is safe to use - ** later on. - ** - ** Unless SQLite is compiled in secure-delete mode. In this case, - ** the dropCell() routine will overwrite the entire cell with zeroes. - ** In this case, temporarily copy the cell into the aOvflSpace[] - ** buffer. It will be copied out again as soon as the aSpace[] buffer - ** is allocated. */ - if( pBt->secureDelete ){ - int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); - if( (iOff+szNew[i])>(int)pBt->usableSize ){ - rc = SQLITE_CORRUPT_BKPT; - memset(apOld, 0, (i+1)*sizeof(MemPage*)); - goto balance_cleanup; - }else{ - memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); - apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; - } - } - dropCell(pParent, i+nxDiv-pParent->nOverflow, szNew[i], &rc); + /* The rollback may have destroyed the pPage1->aData value. So + ** call btreeGetPage() on page 1 again to make + ** sure pPage1->aData is set correctly. */ + if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + int nPage = get4byte(28+(u8*)pPage1->aData); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; + releasePage(pPage1); } + assert( countWriteCursors(pBt)==0 ); + pBt->inTransaction = TRANS_READ; } - /* Make nMaxCells a multiple of 4 in order to preserve 8-byte - ** alignment */ - nMaxCells = (nMaxCells + 3)&~3; - - /* - ** Allocate space for memory structures - */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); - szScratch = - nMaxCells*sizeof(u8*) /* apCell */ - + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ - apCell = sqlite3ScratchMalloc( szScratch ); - if( apCell==0 ){ - rc = SQLITE_NOMEM; - goto balance_cleanup; - } - szCell = (u16*)&apCell[nMaxCells]; - aSpace1 = (u8*)&szCell[nMaxCells]; - assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return rc; +} - /* - ** Load pointers to all cells on sibling pages and the divider cells - ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the the divider Cells - ** from pParent. - ** - ** If the siblings are on leaf pages, then the child pointers of the - ** divider cells are stripped from the cells before they are copied - ** into aSpace1[]. In this way, all cells in apCell[] are without - ** child pointers. If siblings are not leaves, then all cell in - ** apCell[] include child pointers. Either way, all cells in apCell[] - ** are alike. - ** - ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. - ** leafData: 1 if pPage holds key+data and pParent holds only keys. +/* +** Start a statement subtransaction. The subtransaction can can be rolled +** back independently of the main transaction. You must start a transaction +** before starting a subtransaction. The subtransaction is ended automatically +** if the main transaction commits or rolls back. +** +** Statement subtransactions are used around individual SQL statements +** that are contained within a BEGIN...COMMIT block. If a constraint +** error occurs within the statement, the effect of that one statement +** can be rolled back without having to rollback the entire transaction. +** +** A statement sub-transaction is implemented as an anonymous savepoint. The +** value passed as the second parameter is the total number of savepoints, +** including the new anonymous savepoint, open on the B-Tree. i.e. if there +** are no active savepoints and no other statement-transactions open, +** iStatement is 1. This anonymous savepoint can be released or rolled back +** using the sqlite3BtreeSavepoint() function. +*/ +SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( pBt->readOnly==0 ); + assert( iStatement>0 ); + assert( iStatement>p->db->nSavepoint ); + assert( pBt->inTransaction==TRANS_WRITE ); + /* At the pager level, a statement transaction is a savepoint with + ** an index greater than all savepoints created explicitly using + ** SQL statements. It is illegal to open, release or rollback any + ** such savepoints while the statement transaction savepoint is active. */ - leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->hasData; - for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); + sqlite3BtreeLeave(p); + return rc; +} - limit = pOld->nCell+pOld->nOverflow; - for(j=0; jmaxLocal+23 ); - assert( iSpace1<=pBt->pageSize ); - memcpy(pTemp, apDiv[i], sz); - apCell[nCell] = pTemp+leafCorrection; - assert( leafCorrection==0 || leafCorrection==4 ); - szCell[nCell] = szCell[nCell] - leafCorrection; - if( !pOld->leaf ){ - assert( leafCorrection==0 ); - assert( pOld->hdrOffset==0 ); - /* The right pointer of the child page pOld becomes the left - ** pointer of the divider cell */ - memcpy(apCell[nCell], &pOld->aData[8], 4); - }else{ - assert( leafCorrection==4 ); - if( szCell[nCell]<4 ){ - /* Do not allow any cells smaller than 4 bytes. */ - szCell[nCell] = 4; - } - } - nCell++; - } - } +/* +** The second argument to this function, op, is always SAVEPOINT_ROLLBACK +** or SAVEPOINT_RELEASE. This function either releases or rolls back the +** savepoint identified by parameter iSavepoint, depending on the value +** of op. +** +** Normally, iSavepoint is greater than or equal to zero. However, if op is +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** contents of the entire transaction are rolled back. This is different +** from a normal transaction rollback, as no locks are released and the +** transaction remains open. +*/ +SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ + int rc = SQLITE_OK; + if( p && p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); + sqlite3BtreeEnter(p); + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( rc==SQLITE_OK ){ + if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; + rc = newDatabase(pBt); + pBt->nPage = get4byte(28 + pBt->pPage1->aData); - /* - ** Figure out the number of pages needed to hold all nCell cells. - ** Store this number in "k". Also compute szNew[] which is the total - ** size of all cells on the i-th page and cntNew[] which is the index - ** in apCell[] of the cell that divides page i from page i+1. - ** cntNew[k] should equal nCell. - ** - ** Values computed by this block: - ** - ** k: The total number of sibling pages - ** szNew[i]: Spaced used on the i-th sibling page. - ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to - ** the right of the i-th sibling page. - ** usableSpace: Number of bytes of space available on each sibling. - ** - */ - usableSpace = pBt->usableSize - 12 + leafCorrection; - for(subtotal=k=i=0; i usableSpace ){ - szNew[k] = subtotal - szCell[i]; - cntNew[k] = i; - if( leafData ){ i--; } - subtotal = 0; - k++; - if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + /* The database size was written into the offset 28 of the header + ** when the transaction started, so we know that the value at offset + ** 28 is nonzero. */ + assert( pBt->nPage>0 ); } + sqlite3BtreeLeave(p); } - szNew[k] = subtotal; - cntNew[k] = nCell; - k++; + return rc; +} - /* - ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. - ** - ** This adjustment is more than an optimization. The packing above might - ** be so out of balance as to be illegal. For example, the right-most - ** sibling might be completely empty. This adjustment is not optional. - */ - for(i=k-1; i>0; i--){ - int szRight = szNew[i]; /* Size of sibling on the right */ - int szLeft = szNew[i-1]; /* Size of sibling on the left */ - int r; /* Index of right-most cell in left sibling */ - int d; /* Index of first cell to the left of right sibling */ +/* +** Create a new cursor for the BTree whose root is on the page +** iTable. If a read-only cursor is requested, it is assumed that +** the caller already has at least a read-only transaction open +** on the database already. If a write-cursor is requested, then +** the caller is assumed to have an open write transaction. +** +** If wrFlag==0, then the cursor can only be used for reading. +** If wrFlag==1, then the cursor can be used for reading or for +** writing if other conditions for writing are also met. These +** are the conditions that must be met in order for writing to +** be allowed: +** +** 1: The cursor must have been opened with wrFlag==1 +** +** 2: Other database connections that share the same pager cache +** but which are not in the READ_UNCOMMITTED state may not have +** cursors open with wrFlag==0 on the same table. Otherwise +** the changes made by this write cursor would be visible to +** the read cursors in the other database connection. +** +** 3: The database must be writable (not on read-only media) +** +** 4: There must be an active transaction. +** +** No checking is done to make sure that page iTable really is the +** root page of a b-tree. If it is not, then the cursor acquired +** will not work correctly. +** +** It is assumed that the sqlite3BtreeCursorZero() has been called +** on pCur to initialize the memory space prior to invoking this routine. +*/ +static int btreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + BtShared *pBt = p->pBt; /* Shared b-tree handle */ - r = cntNew[i-1] - 1; - d = r + 1 - leafData; - assert( d0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - */ - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with + ** this lock. */ + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, wrFlag+1) ); + assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 - )); + /* Assert that the caller has opened the required transaction. */ + assert( p->inTrans>TRANS_NONE ); + assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1 && pBt->pPage1->aData ); - /* - ** Allocate k new pages. Reuse old pages where possible. - */ - if( apOld[0]->pgno<=1 ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; + if( NEVER(wrFlag && pBt->readOnly) ){ + return SQLITE_READONLY; } - pageFlags = apOld[0]->aData[0]; - for(i=0; ipDbPage); - nNew++; - if( rc ) goto balance_cleanup; - }else{ - assert( i>0 ); - rc = allocateBtreePage(pBt, &pNew, &pgno, pgno, 0); - if( rc ) goto balance_cleanup; - apNew[i] = pNew; - nNew++; - - /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ - ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); - if( rc!=SQLITE_OK ){ - goto balance_cleanup; - } - } - } + if( iTable==1 && btreePagecount(pBt)==0 ){ + return SQLITE_EMPTY; } - /* Free any old pages that were not reused as new pages. - */ - while( ipgnoRoot = (Pgno)iTable; + pCur->iPage = -1; + pCur->pKeyInfo = pKeyInfo; + pCur->pBtree = p; + pCur->pBt = pBt; + pCur->wrFlag = (u8)wrFlag; + pCur->pNext = pBt->pCursor; + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur; } + pBt->pCursor = pCur; + pCur->eState = CURSOR_INVALID; + pCur->cachedRowid = 0; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ + BtCursor *pCur /* Write new cursor here */ +){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} - /* - ** Put the new pages in accending order. This helps to - ** keep entries in the disk file in order so that a scan - ** of the table is a linear scan through the file. That - ** in turn helps the operating system to deliver pages - ** from the disk more rapidly. - ** - ** An O(n^2) insertion sort algorithm is used, but since - ** n is never more than NB (a small constant), that should - ** not be a problem. - ** - ** When NB==3, this one optimization makes the database - ** about 25% faster for large insertions and deletions. - */ - for(i=0; ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; - } - } - if( minI>i ){ - int t; - MemPage *pT; - t = apNew[i]->pgno; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; - } - } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], - nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, - nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, - nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); - - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - put4byte(pRight, apNew[nNew-1]->pgno); - - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. - */ - j = 0; - for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); - - j = cntNew[i]; - - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( ileaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); - } - } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace<=pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); +/* +** Initialize memory that will be converted into a BtCursor object. +** +** The simple approach here would be to memset() the entire object +** to zero. But it turns out that the apPage[] and aiIdx[] arrays +** do not need to be zeroed and they are large, so we can save a lot +** of run-time by skipping the initialization of those elements. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ + memset(p, 0, offsetof(BtCursor, iPage)); +} - j++; - nxDiv++; - } - } - assert( j==nCell ); - assert( nOld>0 ); - assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); +/* +** Set the cached rowid value of every cursor in the same database file +** as pCur and having the same root page number as pCur. The value is +** set to iRowid. +** +** Only positive rowid values are considered valid for this cache. +** The cache is initialized to zero, indicating an invalid cache. +** A btree will work fine with zero or negative rowids. We just cannot +** cache zero or negative rowids, which means tables that use zero or +** negative rowids might run a little slower. But in practice, zero +** or negative rowids are very uncommon so this should not be a problem. +*/ +SQLITE_PRIVATE void sqlite3BtreeSetCachedRowid(BtCursor *pCur, sqlite3_int64 iRowid){ + BtCursor *p; + for(p=pCur->pBt->pCursor; p; p=p->pNext){ + if( p->pgnoRoot==pCur->pgnoRoot ) p->cachedRowid = iRowid; } + assert( pCur->cachedRowid==iRowid ); +} - if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ - /* The root page of the b-tree now contains no cells. The only sibling - ** page is the right-child of the parent. Copy the contents of the - ** child page into the parent, decreasing the overall height of the - ** b-tree structure by one. This is described as the "balance-shallower" - ** sub-algorithm in some documentation. - ** - ** If this is an auto-vacuum database, the call to copyNodeContent() - ** sets all pointer-map entries corresponding to database image pages - ** for which the pointer is stored within the content being copied. - ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ - assert( nNew==1 ); - assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aOvfl[0].idx : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aOvfl[0].idx; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaOvfl[0].idx==pOld->aOvfl[1].idx-1); - assert(nOverflow<3 || pOld->aOvfl[1].idx==pOld->aOvfl[2].idx-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jcachedRowid; +} - /* If the cell was originally divider cell (and is not now) or - ** an overflow cell, or if the cell was located on a different sibling - ** page before the balancing, then the pointer map entries associated - ** with any child or overflow pages need to be updated. */ - if( isDivider || pOld->pgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } +/* +** Close a cursor. The read lock on the database file is released +** when the last cursor is closed. +*/ +SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ + Btree *pBtree = pCur->pBtree; + if( pBtree ){ + int i; + BtShared *pBt = pCur->pBt; + sqlite3BtreeEnter(pBtree); + sqlite3BtreeClearCursor(pCur); + if( pCur->pPrev ){ + pCur->pPrev->pNext = pCur->pNext; + }else{ + pBt->pCursor = pCur->pNext; } - - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur->pPrev; } - -#if 0 - /* The ptrmapCheckPages() contains assert() statements that verify that - ** all pointer map pages are set correctly. This is helpful while - ** debugging. This is usually disabled because a corrupt database may - ** cause an assert() statement to fail. */ - ptrmapCheckPages(apNew, nNew); - ptrmapCheckPages(&pParent, 1); -#endif - } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); - - /* - ** Cleanup before returning. - */ -balance_cleanup: - sqlite3ScratchFree(apCell); - for(i=0; iiPage; i++){ + releasePage(pCur->apPage[i]); + } + unlockBtreeIfUnused(pBt); + invalidateOverflowCache(pCur); + /* sqlite3_free(pCur); */ + sqlite3BtreeLeave(pBtree); } - - return rc; + return SQLITE_OK; } - /* -** This function is called when the root page of a b-tree structure is -** overfull (has one or more overflow pages). -** -** A new child page is allocated and the contents of the current root -** page, including overflow cells, are copied into the child. The root -** page is then overwritten to make it an empty page with the right-child -** pointer pointing to the new page. +** Make sure the BtCursor* given in the argument has a valid +** BtCursor.info structure. If it is not already valid, call +** btreeParseCell() to fill it in. ** -** Before returning, all pointer-map entries corresponding to pages -** that the new child-page now contains pointers to are updated. The -** entry corresponding to the new right-child pointer of the root -** page is also updated. +** BtCursor.info is a cache of the information in the current cell. +** Using this cache reduces the number of calls to btreeParseCell(). ** -** If successful, *ppChild is set to contain a reference to the child -** page and SQLITE_OK is returned. In this case the caller is required -** to call releasePage() on *ppChild exactly once. If an error occurs, -** an error code is returned and *ppChild is set to 0. +** 2007-06-25: There is a bug in some versions of MSVC that cause the +** compiler to crash when getCellInfo() is implemented as a macro. +** But there is a measureable speed advantage to using the macro on gcc +** (when less compiler optimizations like -Os or -O0 are used and the +** compiler is not doing agressive inlining.) So we use a real function +** for MSVC and a macro for everything else. Ticket #2457. */ -static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ - int rc; /* Return value from subprocedures */ - MemPage *pChild = 0; /* Pointer to a new child page */ - Pgno pgnoChild = 0; /* Page number of the new child page */ - BtShared *pBt = pRoot->pBt; /* The BTree */ - - assert( pRoot->nOverflow>0 ); - assert( sqlite3_mutex_held(pBt->mutex) ); - - /* Make pRoot, the root page of the b-tree, writable. Allocate a new - ** page that will become the new right-child of pPage. Copy the contents - ** of the node stored on pRoot into the new child page. - */ - rc = sqlite3PagerWrite(pRoot->pDbPage); - if( rc==SQLITE_OK ){ - rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); - copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ - ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); +#ifndef NDEBUG + static void assertCellInfo(BtCursor *pCur){ + CellInfo info; + int iPage = pCur->iPage; + memset(&info, 0, sizeof(info)); + btreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); + assert( memcmp(&info, &pCur->info, sizeof(info))==0 ); + } +#else + #define assertCellInfo(x) +#endif +#ifdef _MSC_VER + /* Use a real function in MSVC to work around bugs in that compiler. */ + static void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + int iPage = pCur->iPage; + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + pCur->validNKey = 1; + }else{ + assertCellInfo(pCur); } } - if( rc ){ - *ppChild = 0; - releasePage(pChild); - return rc; +#else /* if not _MSC_VER */ + /* Use a macro in all other compilers so that the function is inlined */ +#define getCellInfo(pCur) \ + if( pCur->info.nSize==0 ){ \ + int iPage = pCur->iPage; \ + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ + pCur->validNKey = 1; \ + }else{ \ + assertCellInfo(pCur); \ } - assert( sqlite3PagerIswriteable(pChild->pDbPage) ); - assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - assert( pChild->nCell==pRoot->nCell ); - - TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); +#endif /* _MSC_VER */ - /* Copy the overflow cells from pRoot to pChild */ - memcpy(pChild->aOvfl, pRoot->aOvfl, pRoot->nOverflow*sizeof(pRoot->aOvfl[0])); - pChild->nOverflow = pRoot->nOverflow; +#ifndef NDEBUG /* The next routine used only within assert() statements */ +/* +** Return true if the given BtCursor is valid. A valid cursor is one +** that is currently pointing to a row in a (non-empty) table. +** This is a verification routine is used only within assert() statements. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ + return pCur && pCur->eState==CURSOR_VALID; +} +#endif /* NDEBUG */ - /* Zero the contents of pRoot. Then install pChild as the right-child. */ - zeroPage(pRoot, pChild->aData[0] & ~PTF_LEAF); - put4byte(&pRoot->aData[pRoot->hdrOffset+8], pgnoChild); +/* +** Set *pSize to the size of the buffer needed to hold the value of +** the key for the current entry. If the cursor is not pointing +** to a valid entry, *pSize is set to 0. +** +** For a table with the INTKEY flag set, this routine returns the key +** itself, not the number of bytes in the key. +** +** The caller must position the cursor prior to invoking this routine. +** +** This routine cannot fail. It always returns SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); + if( pCur->eState!=CURSOR_VALID ){ + *pSize = 0; + }else{ + getCellInfo(pCur); + *pSize = pCur->info.nKey; + } + return SQLITE_OK; +} - *ppChild = pChild; +/* +** Set *pSize to the number of bytes of data in the entry the +** cursor currently points to. +** +** The caller must guarantee that the cursor is pointing to a non-NULL +** valid entry. In other words, the calling procedure must guarantee +** that the cursor has Cursor.eState==CURSOR_VALID. +** +** Failure is not possible. This function always returns SQLITE_OK. +** It might just as well be a procedure (returning void) but we continue +** to return an integer result code for historical reasons. +*/ +SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + *pSize = pCur->info.nData; return SQLITE_OK; } /* -** The page that pCur currently points to has just been modified in -** some way. This function figures out if this modification means the -** tree needs to be balanced, and if so calls the appropriate balancing -** routine. Balancing routines are: +** Given the page number of an overflow page in the database (parameter +** ovfl), this function finds the page number of the next page in the +** linked list of overflow pages. If possible, it uses the auto-vacuum +** pointer-map data instead of reading the content of page ovfl to do so. ** -** balance_quick() -** balance_deeper() -** balance_nonroot() +** If an error occurs an SQLite error code is returned. Otherwise: +** +** The page number of the next overflow page in the linked list is +** written to *pPgnoNext. If page ovfl is the last page in its linked +** list, *pPgnoNext is set to zero. +** +** If ppPage is not NULL, and a reference to the MemPage object corresponding +** to page number pOvfl was obtained, then *ppPage is set to point to that +** reference. It is the responsibility of the caller to call releasePage() +** on *ppPage to free the reference. In no reference was obtained (because +** the pointer-map was used to obtain the value for *pPgnoNext), then +** *ppPage is set to zero. */ -static int balance(BtCursor *pCur){ +static int getOverflowPage( + BtShared *pBt, /* The database file */ + Pgno ovfl, /* Current overflow page number */ + MemPage **ppPage, /* OUT: MemPage handle (may be NULL) */ + Pgno *pPgnoNext /* OUT: Next overflow page number */ +){ + Pgno next = 0; + MemPage *pPage = 0; int rc = SQLITE_OK; - const int nMin = pCur->pBt->usableSize * 2 / 3; - u8 aBalanceQuickSpace[13]; - u8 *pFree = 0; - - TESTONLY( int balance_quick_called = 0 ); - TESTONLY( int balance_deeper_called = 0 ); - do { - int iPage = pCur->iPage; - MemPage *pPage = pCur->apPage[iPage]; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert(pPgnoNext); - if( iPage==0 ){ - if( pPage->nOverflow ){ - /* The root page of the b-tree is overfull. In this case call the - ** balance_deeper() function to create a new child for the root-page - ** and copy the current contents of the root-page to it. The - ** next iteration of the do-loop will balance the child page. - */ - assert( (balance_deeper_called++)==0 ); - rc = balance_deeper(pPage, &pCur->apPage[1]); - if( rc==SQLITE_OK ){ - pCur->iPage = 1; - pCur->aiIdx[0] = 0; - pCur->aiIdx[1] = 0; - assert( pCur->apPage[1]->nOverflow ); - } - }else{ - break; - } - }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ - break; - }else{ - MemPage * const pParent = pCur->apPage[iPage-1]; - int const iIdx = pCur->aiIdx[iPage-1]; +#ifndef SQLITE_OMIT_AUTOVACUUM + /* Try to find the next page in the overflow list using the + ** autovacuum pointer-map pages. Guess that the next page in + ** the overflow list is page number (ovfl+1). If that guess turns + ** out to be wrong, fall back to loading the data of page + ** number ovfl to determine the next page number. + */ + if( pBt->autoVacuum ){ + Pgno pgno; + Pgno iGuess = ovfl+1; + u8 eType; - rc = sqlite3PagerWrite(pParent->pDbPage); - if( rc==SQLITE_OK ){ -#ifndef SQLITE_OMIT_QUICKBALANCE - if( pPage->hasData - && pPage->nOverflow==1 - && pPage->aOvfl[0].idx==pPage->nCell - && pParent->pgno!=1 - && pParent->nCell==iIdx - ){ - /* Call balance_quick() to create a new sibling of pPage on which - ** to store the overflow cell. balance_quick() inserts a new cell - ** into pParent, which may cause pParent overflow. If this - ** happens, the next interation of the do-loop will balance pParent - ** use either balance_nonroot() or balance_deeper(). Until this - ** happens, the overflow cell is stored in the aBalanceQuickSpace[] - ** buffer. - ** - ** The purpose of the following assert() is to check that only a - ** single call to balance_quick() is made for each call to this - ** function. If this were not verified, a subtle bug involving reuse - ** of the aBalanceQuickSpace[] might sneak in. - */ - assert( (balance_quick_called++)==0 ); - rc = balance_quick(pParent, pPage, aBalanceQuickSpace); - }else -#endif - { - /* In this case, call balance_nonroot() to redistribute cells - ** between pPage and up to 2 of its sibling pages. This involves - ** modifying the contents of pParent, which may cause pParent to - ** become overfull or underfull. The next iteration of the do-loop - ** will balance the parent page to correct this. - ** - ** If the parent page becomes overfull, the overflow cell or cells - ** are stored in the pSpace buffer allocated immediately below. - ** A subsequent iteration of the do-loop will deal with this by - ** calling balance_nonroot() (balance_deeper() may be called first, - ** but it doesn't deal with overflow cells - just moves them to a - ** different page). Once this subsequent call to balance_nonroot() - ** has completed, it is safe to release the pSpace buffer used by - ** the previous call, as the overflow cell data will have been - ** copied either into the body of a database page or into the new - ** pSpace buffer passed to the latter call to balance_nonroot(). - */ - u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); - rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1); - if( pFree ){ - /* If pFree is not NULL, it points to the pSpace buffer used - ** by a previous call to balance_nonroot(). Its contents are - ** now stored either on real database pages or within the - ** new pSpace buffer, so it may be safely freed here. */ - sqlite3PageFree(pFree); - } + while( PTRMAP_ISPAGE(pBt, iGuess) || iGuess==PENDING_BYTE_PAGE(pBt) ){ + iGuess++; + } - /* The pSpace buffer will be freed after the next call to - ** balance_nonroot(), or just before this function returns, whichever - ** comes first. */ - pFree = pSpace; - } + if( iGuess<=btreePagecount(pBt) ){ + rc = ptrmapGet(pBt, iGuess, &eType, &pgno); + if( rc==SQLITE_OK && eType==PTRMAP_OVERFLOW2 && pgno==ovfl ){ + next = iGuess; + rc = SQLITE_DONE; } + } + } +#endif - pPage->nOverflow = 0; - - /* The next iteration of the do-loop balances the parent page. */ - releasePage(pPage); - pCur->iPage--; + assert( next==0 || rc==SQLITE_DONE ); + if( rc==SQLITE_OK ){ + rc = btreeGetPage(pBt, ovfl, &pPage, 0); + assert( rc==SQLITE_OK || pPage==0 ); + if( rc==SQLITE_OK ){ + next = get4byte(pPage->aData); } - }while( rc==SQLITE_OK ); + } - if( pFree ){ - sqlite3PageFree(pFree); + *pPgnoNext = next; + if( ppPage ){ + *ppPage = pPage; + }else{ + releasePage(pPage); } - return rc; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); } +/* +** Copy data from a buffer to a page, or from a page to a buffer. +** +** pPayload is a pointer to data stored on database page pDbPage. +** If argument eOp is false, then nByte bytes of data are copied +** from pPayload to the buffer pointed at by pBuf. If eOp is true, +** then sqlite3PagerWrite() is called on pDbPage and nByte bytes +** of data are copied from the buffer pBuf to pPayload. +** +** SQLITE_OK is returned on success, otherwise an error code. +*/ +static int copyPayload( + void *pPayload, /* Pointer to page data */ + void *pBuf, /* Pointer to buffer */ + int nByte, /* Number of bytes to copy */ + int eOp, /* 0 -> copy from page, 1 -> copy to page */ + DbPage *pDbPage /* Page containing pPayload */ +){ + if( eOp ){ + /* Copy data from buffer to page (a write operation) */ + int rc = sqlite3PagerWrite(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + memcpy(pPayload, pBuf, nByte); + }else{ + /* Copy data from page to buffer (a read operation) */ + memcpy(pBuf, pPayload, nByte); + } + return SQLITE_OK; +} /* -** Insert a new record into the BTree. The key is given by (pKey,nKey) -** and the data is given by (pData,nData). The cursor is used only to -** define what table the record should be inserted into. The cursor -** is left pointing at a random location. +** This function is used to read or overwrite payload information +** for the entry that the pCur cursor is pointing to. If the eOp +** parameter is 0, this is a read operation (data copied into +** buffer pBuf). If it is non-zero, a write (data copied from +** buffer pBuf). ** -** For an INTKEY table, only the nKey value of the key is used. pKey is -** ignored. For a ZERODATA table, the pData and nData are both ignored. +** A total of "amt" bytes are read or written beginning at "offset". +** Data is read to or from the buffer pBuf. ** -** If the seekResult parameter is non-zero, then a successful call to -** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already -** been performed. seekResult is the search result returned (a negative -** number if pCur points at an entry that is smaller than (pKey, nKey), or -** a positive value if pCur points at an etry that is larger than -** (pKey, nKey)). +** The content being read or written might appear on the main page +** or be scattered out on multiple overflow pages. ** -** If the seekResult parameter is non-zero, then the caller guarantees that -** cursor pCur is pointing at the existing copy of a row that is to be -** overwritten. If the seekResult parameter is 0, then cursor pCur may -** point to any entry or to no entry at all and so this function has to seek -** the cursor before the new key can be inserted. +** If the BtCursor.isIncrblobHandle flag is set, and the current +** cursor entry uses one or more overflow pages, this function +** allocates space for and lazily popluates the overflow page-list +** cache array (BtCursor.aOverflow). Subsequent calls use this +** cache to make seeking to the supplied offset more efficient. +** +** Once an overflow page-list cache has been allocated, it may be +** invalidated if some other cursor writes to the same table, or if +** the cursor is moved to a different row. Additionally, in auto-vacuum +** mode, the following events may invalidate an overflow page-list cache. +** +** * An incremental vacuum, +** * A commit in auto_vacuum="full" mode, +** * Creating a table (may require moving an overflow page). */ -SQLITE_PRIVATE int sqlite3BtreeInsert( - BtCursor *pCur, /* Insert data into the table of this cursor */ - const void *pKey, i64 nKey, /* The key of the new record */ - const void *pData, int nData, /* The data of the new record */ - int nZero, /* Number of extra 0 bytes to append to data */ - int appendBias, /* True if this is likely an append */ - int seekResult /* Result of prior MovetoUnpacked() call */ +static int accessPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 offset, /* Begin reading this far into payload */ + u32 amt, /* Read this many bytes */ + unsigned char *pBuf, /* Write the bytes into this buffer */ + int eOp /* zero to read. non-zero to write. */ ){ - int rc; - int loc = seekResult; /* -1: before desired location +1: after */ - int szNew = 0; - int idx; - MemPage *pPage; - Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; - unsigned char *oldCell; - unsigned char *newCell = 0; - - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; - } + unsigned char *aPayload; + int rc = SQLITE_OK; + u32 nKey; + int iIdx = 0; + MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ + BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ + assert( pPage ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( pCur->wrFlag && pBt->inTransaction==TRANS_WRITE && !pBt->readOnly ); - assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); - /* Assert that the caller has been consistent. If this cursor was opened - ** expecting an index b-tree, then the caller should be inserting blob - ** keys with no associated data. If the cursor was opened expecting an - ** intkey table, the caller should be inserting integer keys with a - ** blob of associated data. */ - assert( (pKey==0)==(pCur->pKeyInfo==0) ); + getCellInfo(pCur); + aPayload = pCur->info.pCell + pCur->info.nHeader; + nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, nKey, 0); + if( NEVER(offset+amt > nKey+pCur->info.nData) + || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ){ + /* Trying to read or write past the end of the data is an error */ + return SQLITE_CORRUPT_BKPT; } - /* Save the positions of any other cursors open on this table. - ** - ** In some cases, the call to btreeMoveto() below is a no-op. For - ** example, when inserting data into a table with auto-generated integer - ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the - ** integer key to use. It then calls this function to actually insert the - ** data into the intkey B-Tree. In this case btreeMoveto() recognizes - ** that the cursor is already where it needs to be and returns without - ** doing any work. To avoid thwarting these optimizations, it is important - ** not to clear the cursor here. - */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; - if( !loc ){ - rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); - if( rc ) return rc; + /* Check if data must be read/written to/from the btree page itself. */ + if( offsetinfo.nLocal ){ + int a = amt; + if( a+offset>pCur->info.nLocal ){ + a = pCur->info.nLocal - offset; + } + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); + offset = 0; + pBuf += a; + amt -= a; + }else{ + offset -= pCur->info.nLocal; } - assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); - pPage = pCur->apPage[pCur->iPage]; - assert( pPage->intKey || nKey>=0 ); - assert( pPage->leaf || !pPage->intKey ); + if( rc==SQLITE_OK && amt>0 ){ + const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ + Pgno nextPage; - TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", - pCur->pgnoRoot, nKey, nData, pPage->pgno, - loc==0 ? "overwrite" : "new entry")); - assert( pPage->isInit ); - allocateTempSpace(pBt); - newCell = pBt->pTmpSpace; - if( newCell==0 ) return SQLITE_NOMEM; - rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); - if( rc ) goto end_insert; - assert( szNew==cellSizePtr(pPage, newCell) ); - assert( szNew<=MX_CELL_SIZE(pBt) ); - idx = pCur->aiIdx[pCur->iPage]; - if( loc==0 ){ - u16 szOld; - assert( idxnCell ); - rc = sqlite3PagerWrite(pPage->pDbPage); - if( rc ){ - goto end_insert; + nextPage = get4byte(&aPayload[pCur->info.nLocal]); + +#ifndef SQLITE_OMIT_INCRBLOB + /* If the isIncrblobHandle flag is set and the BtCursor.aOverflow[] + ** has not been allocated, allocate it now. The array is sized at + ** one entry for each overflow page in the overflow chain. The + ** page number of the first overflow page is stored in aOverflow[0], + ** etc. A value of 0 in the aOverflow[] array means "not yet known" + ** (the cache is lazily populated). + */ + if( pCur->isIncrblobHandle && !pCur->aOverflow ){ + int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; + pCur->aOverflow = (Pgno *)sqlite3MallocZero(sizeof(Pgno)*nOvfl); + /* nOvfl is always positive. If it were zero, fetchPayload would have + ** been used instead of this routine. */ + if( ALWAYS(nOvfl) && !pCur->aOverflow ){ + rc = SQLITE_NOMEM; + } } - oldCell = findCell(pPage, idx); - if( !pPage->leaf ){ - memcpy(newCell, oldCell, 4); + + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow && pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); } - szOld = cellSizePtr(pPage, oldCell); - rc = clearCell(pPage, oldCell); - dropCell(pPage, idx, szOld, &rc); - if( rc ) goto end_insert; - }else if( loc<0 && pPage->nCell>0 ){ - assert( pPage->leaf ); - idx = ++pCur->aiIdx[pCur->iPage]; - }else{ - assert( pPage->leaf ); - } - insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); - assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); +#endif - /* If no error has occured and pPage has an overflow cell, call balance() - ** to redistribute the cells within the tree. Since balance() may move - ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey - ** variables. - ** - ** Previous versions of SQLite called moveToRoot() to move the cursor - ** back to the root page as balance() used to invalidate the contents - ** of BtCursor.apPage[] and BtCursor.aiIdx[]. Instead of doing that, - ** set the cursor state to "invalid". This makes common insert operations - ** slightly faster. - ** - ** There is a subtle but important optimization here too. When inserting - ** multiple records into an intkey b-tree using a single cursor (as can - ** happen while processing an "INSERT INTO ... SELECT" statement), it - ** is advantageous to leave the cursor pointing to the last entry in - ** the b-tree if possible. If the cursor is left pointing to the last - ** entry in the table, and the next row inserted has an integer key - ** larger than the largest existing key, it is possible to insert the - ** row without seeking the cursor. This can be a big performance boost. - */ - pCur->info.nSize = 0; - pCur->validNKey = 0; - if( rc==SQLITE_OK && pPage->nOverflow ){ - rc = balance(pCur); + for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ - /* Must make sure nOverflow is reset to zero even if the balance() - ** fails. Internal data structure corruption will result otherwise. - ** Also, set the cursor state to invalid. This stops saveCursorPosition() - ** from trying to save the current position of the cursor. */ - pCur->apPage[pCur->iPage]->nOverflow = 0; - pCur->eState = CURSOR_INVALID; +#ifndef SQLITE_OMIT_INCRBLOB + /* If required, populate the overflow page-list cache. */ + if( pCur->aOverflow ){ + assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); + pCur->aOverflow[iIdx] = nextPage; + } +#endif + + if( offset>=ovflSize ){ + /* The only reason to read this page is to obtain the page + ** number for the next page in the overflow chain. The page + ** data is not required. So first try to lookup the overflow + ** page-list cache, if any, then fall back to the getOverflowPage() + ** function. + */ +#ifndef SQLITE_OMIT_INCRBLOB + if( pCur->aOverflow && pCur->aOverflow[iIdx+1] ){ + nextPage = pCur->aOverflow[iIdx+1]; + } else +#endif + rc = getOverflowPage(pBt, nextPage, 0, &nextPage); + offset -= ovflSize; + }else{ + /* Need to read this page properly. It contains some of the + ** range of data that is being read (eOp==0) or written (eOp!=0). + */ + DbPage *pDbPage; + int a = amt; + rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + if( rc==SQLITE_OK ){ + aPayload = sqlite3PagerGetData(pDbPage); + nextPage = get4byte(aPayload); + if( a + offset > ovflSize ){ + a = ovflSize - offset; + } + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); + sqlite3PagerUnref(pDbPage); + offset = 0; + amt -= a; + pBuf += a; + } + } + } } - assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); -end_insert: + if( rc==SQLITE_OK && amt>0 ){ + return SQLITE_CORRUPT_BKPT; + } return rc; } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at a arbitrary location. +** Read part of the key associated with cursor pCur. Exactly +** "amt" bytes will be transfered into pBuf[]. The transfer +** begins at "offset". +** +** The caller must ensure that pCur is pointing to a valid row +** in the table. +** +** Return SQLITE_OK on success or an error code if anything goes +** wrong. An error is returned if "offset+amt" is larger than +** the available payload. */ -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ - Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; - int rc; /* Return code */ - MemPage *pPage; /* Page to delete cell from */ - unsigned char *pCell; /* Pointer to cell to delete */ - int iCellIdx; /* Index of cell to delete */ - int iCellDepth; /* Depth of node containing pCell */ - +SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ assert( cursorHoldsMutex(pCur) ); - assert( pBt->inTransaction==TRANS_WRITE ); - assert( !pBt->readOnly ); - assert( pCur->wrFlag ); - assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); - assert( !hasReadConflicts(p, pCur->pgnoRoot) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); +} - if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) - || NEVER(pCur->eState!=CURSOR_VALID) - ){ - return SQLITE_ERROR; /* Something has gone awry. */ - } +/* +** Read part of the data associated with cursor pCur. Exactly +** "amt" bytes will be transfered into pBuf[]. The transfer +** begins at "offset". +** +** Return SQLITE_OK on success or an error code if anything goes +** wrong. An error is returned if "offset+amt" is larger than +** the available payload. +*/ +SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + int rc; - /* If this is a delete operation to remove a row from a table b-tree, - ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); +#ifndef SQLITE_OMIT_INCRBLOB + if ( pCur->eState==CURSOR_INVALID ){ + return SQLITE_ABORT; } +#endif - iCellDepth = pCur->iPage; - iCellIdx = pCur->aiIdx[iCellDepth]; - pPage = pCur->apPage[iCellDepth]; - pCell = findCell(pPage, iCellIdx); - - /* If the page containing the entry to delete is not a leaf page, move - ** the cursor to the largest entry in the tree that is smaller than - ** the entry being deleted. This cell will replace the cell being deleted - ** from the internal node. The 'previous' entry is used for this instead - ** of the 'next' entry, as the previous entry is always a part of the - ** sub-tree headed by the child page of the cell being deleted. This makes - ** balancing the tree following the delete operation easier. */ - if( !pPage->leaf ){ - int notUsed; - rc = sqlite3BtreePrevious(pCur, ¬Used); - if( rc ) return rc; + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + rc = accessPayload(pCur, offset, amt, pBuf, 0); } + return rc; +} - /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; - rc = sqlite3PagerWrite(pPage->pDbPage); - if( rc ) return rc; - rc = clearCell(pPage, pCell); - dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); - if( rc ) return rc; - - /* If the cell deleted was not located on a leaf page, then the cursor - ** is currently pointing to the largest entry in the sub-tree headed - ** by the child-page of the cell that was just deleted from an internal - ** node. The cell from the leaf node needs to be moved to the internal - ** node to replace the deleted cell. */ - if( !pPage->leaf ){ - MemPage *pLeaf = pCur->apPage[pCur->iPage]; - int nCell; - Pgno n = pCur->apPage[iCellDepth+1]->pgno; - unsigned char *pTmp; - - pCell = findCell(pLeaf, pLeaf->nCell-1); - nCell = cellSizePtr(pLeaf, pCell); - assert( MX_CELL_SIZE(pBt)>=nCell ); - - allocateTempSpace(pBt); - pTmp = pBt->pTmpSpace; +/* +** Return a pointer to payload information from the entry that the +** pCur cursor is pointing to. The pointer is to the beginning of +** the key if skipKey==0 and it points to the beginning of data if +** skipKey==1. The number of bytes of available key/data is written +** into *pAmt. If *pAmt==0, then the value returned will not be +** a valid pointer. +** +** This routine is an optimization. It is common for the entire key +** and data to fit on the local page and for there to be no overflow +** pages. When that is so, this routine can be used to access the +** key and data without making a copy. If the key and/or data spills +** onto overflow pages, then accessPayload() must be used to reassemble +** the key/data and copy it into a preallocated buffer. +** +** The pointer returned by this routine looks directly into the cached +** page of the database. The data might change or move the next time +** any btree routine is called. +*/ +static const unsigned char *fetchPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + int *pAmt, /* Write the number of available bytes here */ + int skipKey /* read beginning at data if this is true */ +){ + unsigned char *aPayload; + MemPage *pPage; + u32 nKey; + u32 nLocal; - rc = sqlite3PagerWrite(pLeaf->pDbPage); - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); - dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); - if( rc ) return rc; + assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); + assert( pCur->eState==CURSOR_VALID ); + assert( cursorHoldsMutex(pCur) ); + pPage = pCur->apPage[pCur->iPage]; + assert( pCur->aiIdx[pCur->iPage]nCell ); + if( NEVER(pCur->info.nSize==0) ){ + btreeParseCell(pCur->apPage[pCur->iPage], pCur->aiIdx[pCur->iPage], + &pCur->info); } - - /* Balance the tree. If the entry deleted was located on a leaf page, - ** then the cursor still points to that page. In this case the first - ** call to balance() repairs the tree, and the if(...) condition is - ** never true. - ** - ** Otherwise, if the entry deleted was on an internal node page, then - ** pCur is pointing to the leaf page from which a cell was removed to - ** replace the cell deleted from the internal node. This is slightly - ** tricky as the leaf node may be underfull, and the internal node may - ** be either under or overfull. In this case run the balancing algorithm - ** on the leaf node first. If the balance proceeds far enough up the - ** tree that we can be sure that any problem in the internal node has - ** been corrected, so be it. Otherwise, after balancing the leaf node, - ** walk the cursor up the tree to the internal node and balance it as - ** well. */ - rc = balance(pCur); - if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ - while( pCur->iPage>iCellDepth ){ - releasePage(pCur->apPage[pCur->iPage--]); - } - rc = balance(pCur); + aPayload = pCur->info.pCell; + aPayload += pCur->info.nHeader; + if( pPage->intKey ){ + nKey = 0; + }else{ + nKey = (int)pCur->info.nKey; } - - if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( skipKey ){ + aPayload += nKey; + nLocal = pCur->info.nLocal - nKey; + }else{ + nLocal = pCur->info.nLocal; + assert( nLocal<=nKey ); } - return rc; + *pAmt = nLocal; + return aPayload; } + /* -** Create a new BTree table. Write into *piTable the page -** number for the root page of the new table. +** For the entry that cursor pCur is point to, return as +** many bytes of the key or data as are available on the local +** b-tree page. Write the number of available bytes into *pAmt. ** -** The type of type is determined by the flags parameter. Only the -** following values of flags are currently in use. Other values for -** flags might not work: +** The pointer returned is ephemeral. The key/data may move +** or be destroyed on the next call to any Btree routine, +** including calls from other threads against the same cache. +** Hence, a mutex on the BtShared should be held prior to calling +** this routine. ** -** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys -** BTREE_ZERODATA Used for SQL indices +** These routines is used to get quick access to key and data +** in the common case where no overflow pages are used. */ -static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ - BtShared *pBt = p->pBt; - MemPage *pRoot; - Pgno pgnoRoot; +SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor *pCur, int *pAmt){ + const void *p = 0; + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( cursorHoldsMutex(pCur) ); + if( ALWAYS(pCur->eState==CURSOR_VALID) ){ + p = (const void*)fetchPayload(pCur, pAmt, 0); + } + return p; +} +SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, int *pAmt){ + const void *p = 0; + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( cursorHoldsMutex(pCur) ); + if( ALWAYS(pCur->eState==CURSOR_VALID) ){ + p = (const void*)fetchPayload(pCur, pAmt, 1); + } + return p; +} + + +/* +** Move the cursor down to a new child page. The newPgno argument is the +** page number of the child page to move to. +** +** This function returns SQLITE_CORRUPT if the page-header flags field of +** the new child page does not match the flags field of the parent (i.e. +** if an intkey page appears to be the parent of a non-intkey page, or +** vice-versa). +*/ +static int moveToChild(BtCursor *pCur, u32 newPgno){ int rc; - int ptfFlags; /* Page-type flage for the root page of new table */ + int i = pCur->iPage; + MemPage *pNewPage; + BtShared *pBt = pCur->pBt; - assert( sqlite3BtreeHoldsMutex(p) ); - assert( pBt->inTransaction==TRANS_WRITE ); - assert( !pBt->readOnly ); + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPageiPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = getAndInitPage(pBt, newPgno, &pNewPage); + if( rc ) return rc; + pCur->apPage[i+1] = pNewPage; + pCur->aiIdx[i+1] = 0; + pCur->iPage++; -#ifdef SQLITE_OMIT_AUTOVACUUM - rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); - if( rc ){ - return rc; + pCur->info.nSize = 0; + pCur->validNKey = 0; + if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ + return SQLITE_CORRUPT_BKPT; + } + return SQLITE_OK; +} + +#ifndef NDEBUG +/* +** Page pParent is an internal (non-leaf) tree page. This function +** asserts that page number iChild is the left-child if the iIdx'th +** cell in page pParent. Or, if iIdx is equal to the total number of +** cells in pParent, that page number iChild is the right-child of +** the page. +*/ +static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + assert( iIdx<=pParent->nCell ); + if( iIdx==pParent->nCell ){ + assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); + }else{ + assert( get4byte(findCell(pParent, iIdx))==iChild ); } +} #else - if( pBt->autoVacuum ){ - Pgno pgnoMove; /* Move a page here to make room for the root-page */ - MemPage *pPageMove; /* The page to move to. */ +# define assertParentIndex(x,y,z) +#endif - /* Creating a new table may probably require moving an existing database - ** to make room for the new tables root page. In case this page turns - ** out to be an overflow page, delete all overflow page-map caches - ** held by open cursors. - */ - invalidateAllOverflowCache(pBt); +/* +** Move the cursor up to the parent page. +** +** pCur->idx is set to the cell index that contains the pointer +** to the page we are coming from. If we are coming from the +** right-most child page then pCur->idx is set to one more than +** the largest cell index. +*/ +static void moveToParent(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>0 ); + assert( pCur->apPage[pCur->iPage] ); + assertParentIndex( + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], + pCur->apPage[pCur->iPage]->pgno + ); + releasePage(pCur->apPage[pCur->iPage]); + pCur->iPage--; + pCur->info.nSize = 0; + pCur->validNKey = 0; +} - /* Read the value of meta[3] from the database to determine where the - ** root page of the new table should go. meta[3] is the largest root-page - ** created so far, so the new root-page is (meta[3]+1). - */ - sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); - pgnoRoot++; +/* +** Move the cursor to point to the root page of its b-tree structure. +** +** If the table has a virtual root page, then the cursor is moved to point +** to the virtual root page instead of the actual root page. A table has a +** virtual root page when the actual root page contains no cells and a +** single child page. This can only happen with the table rooted at page 1. +** +** If the b-tree structure is empty, the cursor state is set to +** CURSOR_INVALID. Otherwise, the cursor is set to point to the first +** cell located on the root (or virtual root) page and the cursor state +** is set to CURSOR_VALID. +** +** If this function returns successfully, it may be assumed that the +** page-header flags indicate that the [virtual] root-page is the expected +** kind of b-tree page (i.e. if when opening the cursor the caller did not +** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, +** indicating a table b-tree, or if the caller did specify a KeyInfo +** structure the flags byte is set to 0x02 or 0x0A, indicating an index +** b-tree). +*/ +static int moveToRoot(BtCursor *pCur){ + MemPage *pRoot; + int rc = SQLITE_OK; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; - /* The new root-page may not be allocated on a pointer-map page, or the - ** PENDING_BYTE page. - */ - while( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || - pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ - pgnoRoot++; + assert( cursorHoldsMutex(pCur) ); + assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; } - assert( pgnoRoot>=3 ); + sqlite3BtreeClearCursor(pCur); + } - /* Allocate a page. The page that currently resides at pgnoRoot will - ** be moved to the allocated page (unless the allocated page happens - ** to reside at pgnoRoot). - */ - rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); + if( pCur->iPage>=0 ){ + int i; + for(i=1; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + } + pCur->iPage = 0; + }else{ + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); if( rc!=SQLITE_OK ){ + pCur->eState = CURSOR_INVALID; return rc; } + pCur->iPage = 0; - if( pgnoMove!=pgnoRoot ){ - /* pgnoRoot is the page that will be used for the root-page of - ** the new table (assuming an error did not occur). But we were - ** allocated pgnoMove. If required (i.e. if it was not allocated - ** by extending the file), the current page at position pgnoMove - ** is already journaled. - */ - u8 eType = 0; - Pgno iPtrPage = 0; - - releasePage(pPageMove); - - /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); - if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; - } - if( rc!=SQLITE_OK ){ - releasePage(pRoot); - return rc; - } - assert( eType!=PTRMAP_ROOTPAGE ); - assert( eType!=PTRMAP_FREEPAGE ); - rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove, 0); - releasePage(pRoot); - - /* Obtain the page at pgnoRoot */ - if( rc!=SQLITE_OK ){ - return rc; - } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3PagerWrite(pRoot->pDbPage); - if( rc!=SQLITE_OK ){ - releasePage(pRoot); - return rc; - } - }else{ - pRoot = pPageMove; - } - - /* Update the pointer-map and meta-data with the new root-page number. */ - ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); - if( rc ){ - releasePage(pRoot); - return rc; + /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor + ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is + ** NULL, the caller expects a table b-tree. If this is not the case, + ** return an SQLITE_CORRUPT error. */ + assert( pCur->apPage[0]->intKey==1 || pCur->apPage[0]->intKey==0 ); + if( (pCur->pKeyInfo==0)!=pCur->apPage[0]->intKey ){ + return SQLITE_CORRUPT_BKPT; } + } - /* When the new root page was allocated, page 1 was made writable in - ** order either to increase the database filesize, or to decrement the - ** freelist count. Hence, the sqlite3BtreeUpdateMeta() call cannot fail. - */ - assert( sqlite3PagerIswriteable(pBt->pPage1->pDbPage) ); - rc = sqlite3BtreeUpdateMeta(p, 4, pgnoRoot); - if( NEVER(rc) ){ - releasePage(pRoot); - return rc; - } + /* Assert that the root page is of the correct type. This must be the + ** case as the call to this function that loaded the root-page (either + ** this call or a previous invocation) would have detected corruption + ** if the assumption were not true, and it is not possible for the flags + ** byte to have been modified while this cursor is holding a reference + ** to the page. */ + pRoot = pCur->apPage[0]; + assert( pRoot->pgno==pCur->pgnoRoot ); + assert( pRoot->isInit && (pCur->pKeyInfo==0)==pRoot->intKey ); + pCur->aiIdx[0] = 0; + pCur->info.nSize = 0; + pCur->atLast = 0; + pCur->validNKey = 0; + + if( pRoot->nCell==0 && !pRoot->leaf ){ + Pgno subpage; + if( pRoot->pgno!=1 ) return SQLITE_CORRUPT_BKPT; + subpage = get4byte(&pRoot->aData[pRoot->hdrOffset+8]); + pCur->eState = CURSOR_VALID; + rc = moveToChild(pCur, subpage); }else{ - rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); - if( rc ) return rc; - } -#endif - assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - if( createTabFlags & BTREE_INTKEY ){ - ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; - }else{ - ptfFlags = PTF_ZERODATA | PTF_LEAF; + pCur->eState = ((pRoot->nCell>0)?CURSOR_VALID:CURSOR_INVALID); } - zeroPage(pRoot, ptfFlags); - sqlite3PagerUnref(pRoot->pDbPage); - assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); - *piTable = (int)pgnoRoot; - return SQLITE_OK; -} -SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ - int rc; - sqlite3BtreeEnter(p); - rc = btreeCreateTable(p, piTable, flags); - sqlite3BtreeLeave(p); return rc; } /* -** Erase the given database page and all its children. Return -** the page to the freelist. +** Move the cursor down to the left-most leaf entry beneath the +** entry to which it is currently pointing. +** +** The left-most leaf is the one with the smallest key - the first +** in ascending order. */ -static int clearDatabasePage( - BtShared *pBt, /* The BTree that contains the table */ - Pgno pgno, /* Page number to clear */ - int freePageFlag, /* Deallocate page if true */ - int *pnChange /* Add number of Cells freed to this counter */ -){ +static int moveToLeftmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; MemPage *pPage; - int rc; - unsigned char *pCell; - int i; - - assert( sqlite3_mutex_held(pBt->mutex) ); - if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; - } - rc = getAndInitPage(pBt, pgno, &pPage); - if( rc ) return rc; - for(i=0; inCell; i++){ - pCell = findCell(pPage, i); - if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); - if( rc ) goto cleardatabasepage_out; - } - rc = clearCell(pPage, pCell); - if( rc ) goto cleardatabasepage_out; - } - if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange); - if( rc ) goto cleardatabasepage_out; - }else if( pnChange ){ - assert( pPage->intKey ); - *pnChange += pPage->nCell; - } - if( freePageFlag ){ - freePage(pPage, &rc); - }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ - zeroPage(pPage, pPage->aData[0] | PTF_LEAF); + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + assert( pCur->aiIdx[pCur->iPage]nCell ); + pgno = get4byte(findCell(pPage, pCur->aiIdx[pCur->iPage])); + rc = moveToChild(pCur, pgno); } - -cleardatabasepage_out: - releasePage(pPage); return rc; } /* -** Delete all information from a single table in the database. iTable is -** the page number of the root of the table. After this routine returns, -** the root page is empty, but still exists. -** -** This routine will fail with SQLITE_LOCKED if there are any open -** read cursors on the table. Open write cursors are moved to the -** root of the table. +** Move the cursor down to the right-most leaf entry beneath the +** page to which it is currently pointing. Notice the difference +** between moveToLeftmost() and moveToRightmost(). moveToLeftmost() +** finds the left-most entry beneath the *entry* whereas moveToRightmost() +** finds the right-most entry beneath the *page*. ** -** If pnChange is not NULL, then table iTable must be an intkey table. The -** integer value pointed to by pnChange is incremented by the number of -** entries in the table. +** The right-most entry is the one with the largest key - the last +** key in ascending order. */ -SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ - int rc; - BtShared *pBt = p->pBt; - sqlite3BtreeEnter(p); - assert( p->inTrans==TRANS_WRITE ); - - /* Invalidate all incrblob cursors open on table iTable (assuming iTable - ** is the root of a table b-tree - if it is not, the following call is - ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); +static int moveToRightmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage = 0; - rc = saveAllCursors(pBt, (Pgno)iTable, 0); - if( SQLITE_OK==rc ){ - rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + pCur->aiIdx[pCur->iPage] = pPage->nCell; + rc = moveToChild(pCur, pgno); + } + if( rc==SQLITE_OK ){ + pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + pCur->info.nSize = 0; + pCur->validNKey = 0; } - sqlite3BtreeLeave(p); return rc; } -/* -** Erase all information in a table and add the root of the table to -** the freelist. Except, the root of the principle table (the one on -** page 1) is never added to the freelist. +/* Move the cursor to the first entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + if( pCur->eState==CURSOR_INVALID ){ + assert( pCur->apPage[pCur->iPage]->nCell==0 ); + *pRes = 1; + }else{ + assert( pCur->apPage[pCur->iPage]->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + } + } + return rc; +} + +/* Move the cursor to the last entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* If the cursor already points to the last entry, this is a no-op. */ + if( CURSOR_VALID==pCur->eState && pCur->atLast ){ +#ifdef SQLITE_DEBUG + /* This block serves to assert() that the cursor really does point + ** to the last entry in the b-tree. */ + int ii; + for(ii=0; iiiPage; ii++){ + assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); + } + assert( pCur->aiIdx[pCur->iPage]==pCur->apPage[pCur->iPage]->nCell-1 ); + assert( pCur->apPage[pCur->iPage]->leaf ); +#endif + return SQLITE_OK; + } + + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + if( CURSOR_INVALID==pCur->eState ){ + assert( pCur->apPage[pCur->iPage]->nCell==0 ); + *pRes = 1; + }else{ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + pCur->atLast = rc==SQLITE_OK ?1:0; + } + } + return rc; +} + +/* Move the cursor so that it points to an entry near the key +** specified by pIdxKey or intKey. Return a success code. ** -** This routine will fail with SQLITE_LOCKED if there are any open -** cursors on the table. +** For INTKEY tables, the intKey parameter is used. pIdxKey +** must be NULL. For index tables, pIdxKey is used and intKey +** is ignored. +** +** If an exact match is not found, then the cursor is always +** left pointing at a leaf page which would hold the entry if it +** were present. The cursor might point to an entry that comes +** before or after the key. +** +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: +** +** *pRes<0 The cursor is left pointing at an entry that +** is smaller than intKey/pIdxKey or if the table is empty +** and the cursor is therefore left point to nothing. +** +** *pRes==0 The cursor is left pointing at an entry that +** exactly matches intKey/pIdxKey. +** +** *pRes>0 The cursor is left pointing at an entry that +** is larger than intKey/pIdxKey. ** -** If AUTOVACUUM is enabled and the page at iTable is not the last -** root page in the database file, then the last root page -** in the database file is moved into the slot formerly occupied by -** iTable and that last slot formerly occupied by the last root page -** is added to the freelist instead of iTable. In this say, all -** root pages are kept at the beginning of the database file, which -** is necessary for AUTOVACUUM to work right. *piMoved is set to the -** page number that used to be the last root page in the file before -** the move. If no page gets moved, *piMoved is set to 0. -** The last root page is recorded in meta[3] and the value of -** meta[3] is updated by this procedure. */ -static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ +SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( + BtCursor *pCur, /* The cursor to be moved */ + UnpackedRecord *pIdxKey, /* Unpacked index key */ + i64 intKey, /* The table key */ + int biasRight, /* If true, bias the search to the high end */ + int *pRes /* Write search results here */ +){ int rc; - MemPage *pPage = 0; - BtShared *pBt = p->pBt; - assert( sqlite3BtreeHoldsMutex(p) ); - assert( p->inTrans==TRANS_WRITE ); + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( pRes ); + assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); - /* It is illegal to drop a table if any cursors are open on the - ** database. This is because in auto-vacuum mode the backend may - ** need to move another root-page to fill a gap left by the deleted - ** root page. If an open cursor was using this page a problem would - ** occur. - ** - ** This error is caught long before control reaches this point. - */ - if( NEVER(pBt->pCursor) ){ - sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db); - return SQLITE_LOCKED_SHAREDCACHE; + /* If the cursor is already positioned at the point we are trying + ** to move to, then just return without doing any work */ + if( pCur->eState==CURSOR_VALID && pCur->validNKey + && pCur->apPage[0]->intKey + ){ + if( pCur->info.nKey==intKey ){ + *pRes = 0; + return SQLITE_OK; + } + if( pCur->atLast && pCur->info.nKeyapPage[pCur->iPage] ); + assert( pCur->apPage[pCur->iPage]->isInit ); + assert( pCur->apPage[pCur->iPage]->nCell>0 || pCur->eState==CURSOR_INVALID ); + if( pCur->eState==CURSOR_INVALID ){ + *pRes = -1; + assert( pCur->apPage[pCur->iPage]->nCell==0 ); + return SQLITE_OK; + } + assert( pCur->apPage[0]->intKey || pIdxKey ); + for(;;){ + int lwr, upr, idx; + Pgno chldPg; + MemPage *pPage = pCur->apPage[pCur->iPage]; + int c; - *piMoved = 0; - - if( iTable>1 ){ -#ifdef SQLITE_OMIT_AUTOVACUUM - freePage(pPage, &rc); - releasePage(pPage); -#else - if( pBt->autoVacuum ){ - Pgno maxRootPgno; - sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + /* pPage->nCell must be greater than zero. If this is the root-page + ** the cursor would have been INVALID above and this for(;;) loop + ** not run. If this is not the root-page, then the moveToChild() routine + ** would have already detected db corruption. Similarly, pPage must + ** be the right kind (index or table) of b-tree page. Otherwise + ** a moveToChild() or moveToRoot() call would have detected corruption. */ + assert( pPage->nCell>0 ); + assert( pPage->intKey==(pIdxKey==0) ); + lwr = 0; + upr = pPage->nCell-1; + if( biasRight ){ + pCur->aiIdx[pCur->iPage] = (u16)(idx = upr); + }else{ + pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2); + } + for(;;){ + u8 *pCell; /* Pointer to current cell in pPage */ - if( iTable==maxRootPgno ){ - /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. - */ - freePage(pPage, &rc); - releasePage(pPage); - if( rc!=SQLITE_OK ){ - return rc; + assert( idx==pCur->aiIdx[pCur->iPage] ); + pCur->info.nSize = 0; + pCell = findCell(pPage, idx) + pPage->childPtrSize; + if( pPage->intKey ){ + i64 nCellKey; + if( pPage->hasData ){ + u32 dummy; + pCell += getVarint32(pCell, dummy); } + getVarint(pCell, (u64*)&nCellKey); + if( nCellKey==intKey ){ + c = 0; + }else if( nCellKeyintKey ); + c = +1; + } + pCur->validNKey = 1; + pCur->info.nKey = nCellKey; }else{ - /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the - ** gap left by the deleted root-page. + /* The maximum supported page-size is 65536 bytes. This means that + ** the maximum number of record bytes stored on an index B-Tree + ** page is less than 16384 bytes and may be stored as a 2-byte + ** varint. This information is used to attempt to avoid parsing + ** the entire cell by checking for the cases where the record is + ** stored entirely within the b-tree page by inspecting the first + ** 2 bytes of the cell. */ - MemPage *pMove; - releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; + int nCell = pCell[0]; + if( !(nCell & 0x80) && nCell<=pPage->maxLocal ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + }else{ + /* The record flows over onto one or more overflow pages. In + ** this case the whole cell needs to be parsed, a buffer allocated + ** and accessPayload() used to retrieve the record into the + ** buffer before VdbeRecordCompare() can be called. */ + void *pCellKey; + u8 * const pCellBody = pCell - pPage->childPtrSize; + btreeParseCellPtr(pPage, pCellBody, &pCur->info); + nCell = (int)pCur->info.nKey; + pCellKey = sqlite3Malloc( nCell ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM; + goto moveto_finish; + } + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + if( rc ){ + sqlite3_free(pCellKey); + goto moveto_finish; + } + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); + sqlite3_free(pCellKey); } - pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); - freePage(pMove, &rc); - releasePage(pMove); - if( rc!=SQLITE_OK ){ - return rc; + } + if( c==0 ){ + if( pPage->intKey && !pPage->leaf ){ + lwr = idx; + upr = lwr - 1; + break; + }else{ + *pRes = 0; + rc = SQLITE_OK; + goto moveto_finish; } - *piMoved = maxRootPgno; } - - /* Set the new 'max-root-page' value in the database header. This - ** is the old value less one, less one more if that happens to - ** be a root-page number, less one again if that is the - ** PENDING_BYTE_PAGE. - */ - maxRootPgno--; - while( maxRootPgno==PENDING_BYTE_PAGE(pBt) - || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ - maxRootPgno--; + if( c<0 ){ + lwr = idx+1; + }else{ + upr = idx-1; } - assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - - rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + if( lwr>upr ){ + break; + } + pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); + } + assert( lwr==upr+1 ); + assert( pPage->isInit ); + if( pPage->leaf ){ + chldPg = 0; + }else if( lwr>=pPage->nCell ){ + chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); }else{ - freePage(pPage, &rc); - releasePage(pPage); + chldPg = get4byte(findCell(pPage, lwr)); } -#endif - }else{ - /* If sqlite3BtreeDropTable was called on page 1. - ** This really never should happen except in a corrupt - ** database. - */ - zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); - releasePage(pPage); + if( chldPg==0 ){ + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + *pRes = c; + rc = SQLITE_OK; + goto moveto_finish; + } + pCur->aiIdx[pCur->iPage] = (u16)lwr; + pCur->info.nSize = 0; + pCur->validNKey = 0; + rc = moveToChild(pCur, chldPg); + if( rc ) goto moveto_finish; } - return rc; -} -SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ - int rc; - sqlite3BtreeEnter(p); - rc = btreeDropTable(p, iTable, piMoved); - sqlite3BtreeLeave(p); +moveto_finish: return rc; } /* -** This function may only be called if the b-tree connection already -** has a read or write transaction open on the database. +** Return TRUE if the cursor is not pointing at an entry of the table. ** -** Read the meta-information out of a database file. Meta[0] -** is the number of free pages currently in the database. Meta[1] -** through meta[15] are available for use by higher layers. Meta[0] -** is read-only, the others are read/write. -** -** The schema layer numbers meta values differently. At the schema -** layer (and the SetCookie and ReadCookie opcodes) the number of -** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** TRUE will be returned after a call to sqlite3BtreeNext() moves +** past the last entry in the table or sqlite3BtreePrev() moves past +** the first entry. TRUE is also returned if the table is empty. */ -SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ - BtShared *pBt = p->pBt; - - sqlite3BtreeEnter(p); - assert( p->inTrans>TRANS_NONE ); - assert( SQLITE_OK==querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK) ); - assert( pBt->pPage1 ); - assert( idx>=0 && idx<=15 ); - - *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); - - /* If auto-vacuum is disabled in this build and this is an auto-vacuum - ** database, mark the database as read-only. */ -#ifdef SQLITE_OMIT_AUTOVACUUM - if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ) pBt->readOnly = 1; -#endif - - sqlite3BtreeLeave(p); +SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ + /* TODO: What if the cursor is in CURSOR_REQUIRESEEK but all table entries + ** have been deleted? This API will need to change to return an error code + ** as well as the boolean result value. + */ + return (CURSOR_VALID!=pCur->eState); } /* -** Write meta-information back into the database. Meta[0] is -** read-only and may not be written. +** Advance the cursor to the next entry in the database. If +** successful then set *pRes=0. If the cursor +** was already pointing to the last entry in the database before +** this routine was called, then set *pRes=1. */ -SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ - BtShared *pBt = p->pBt; - unsigned char *pP1; +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ int rc; - assert( idx>=1 && idx<=15 ); - sqlite3BtreeEnter(p); - assert( p->inTrans==TRANS_WRITE ); - assert( pBt->pPage1!=0 ); - pP1 = pBt->pPage1->aData; - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - if( rc==SQLITE_OK ){ - put4byte(&pP1[36 + idx*4], iMeta); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( idx==BTREE_INCR_VACUUM ){ - assert( pBt->autoVacuum || iMeta==0 ); - assert( iMeta==0 || iMeta==1 ); - pBt->incrVacuum = (u8)iMeta; + int idx; + MemPage *pPage; + + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( pRes!=0 ); + if( CURSOR_INVALID==pCur->eState ){ + *pRes = 1; + return SQLITE_OK; + } + if( pCur->skipNext>0 ){ + pCur->skipNext = 0; + *pRes = 0; + return SQLITE_OK; + } + pCur->skipNext = 0; + + pPage = pCur->apPage[pCur->iPage]; + idx = ++pCur->aiIdx[pCur->iPage]; + assert( pPage->isInit ); + assert( idx<=pPage->nCell ); + + pCur->info.nSize = 0; + pCur->validNKey = 0; + if( idx>=pPage->nCell ){ + if( !pPage->leaf ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + if( rc ) return rc; + rc = moveToLeftmost(pCur); + *pRes = 0; + return rc; } -#endif + do{ + if( pCur->iPage==0 ){ + *pRes = 1; + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; + } + moveToParent(pCur); + pPage = pCur->apPage[pCur->iPage]; + }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); + *pRes = 0; + if( pPage->intKey ){ + rc = sqlite3BtreeNext(pCur, pRes); + }else{ + rc = SQLITE_OK; + } + return rc; } - sqlite3BtreeLeave(p); + *pRes = 0; + if( pPage->leaf ){ + return SQLITE_OK; + } + rc = moveToLeftmost(pCur); return rc; } -#ifndef SQLITE_OMIT_BTREECOUNT + /* -** The first argument, pCur, is a cursor opened on some b-tree. Count the -** number of entries in the b-tree and write the result to *pnEntry. -** -** SQLITE_OK is returned if the operation is successfully executed. -** Otherwise, if an error is encountered (i.e. an IO error or database -** corruption) an SQLite error code is returned. +** Step the cursor to the back to the previous entry in the database. If +** successful then set *pRes=0. If the cursor +** was already pointing to the first entry in the database before +** this routine was called, then set *pRes=1. */ -SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ - i64 nEntry = 0; /* Value to return in *pnEntry */ - int rc; /* Return code */ - rc = moveToRoot(pCur); +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ + int rc; + MemPage *pPage; - /* Unless an error occurs, the following loop runs one iteration for each - ** page in the B-Tree structure (not including overflow pages). - */ - while( rc==SQLITE_OK ){ - int iIdx; /* Index of child node in parent */ - MemPage *pPage; /* Current page of the b-tree */ + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; + } + pCur->atLast = 0; + if( CURSOR_INVALID==pCur->eState ){ + *pRes = 1; + return SQLITE_OK; + } + if( pCur->skipNext<0 ){ + pCur->skipNext = 0; + *pRes = 0; + return SQLITE_OK; + } + pCur->skipNext = 0; - /* If this is a leaf page or the tree is not an int-key tree, then - ** this page contains countable entries. Increment the entry counter - ** accordingly. - */ - pPage = pCur->apPage[pCur->iPage]; - if( pPage->leaf || !pPage->intKey ){ - nEntry += pPage->nCell; + pPage = pCur->apPage[pCur->iPage]; + assert( pPage->isInit ); + if( !pPage->leaf ){ + int idx = pCur->aiIdx[pCur->iPage]; + rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); + if( rc ){ + return rc; } - - /* pPage is a leaf node. This loop navigates the cursor so that it - ** points to the first interior cell that it points to the parent of - ** the next page in the tree that has not yet been visited. The - ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell - ** of the page, or to the number of cells in the page if the next page - ** to visit is the right-child of its parent. - ** - ** If all pages in the tree have been visited, return SQLITE_OK to the - ** caller. - */ - if( pPage->leaf ){ - do { - if( pCur->iPage==0 ){ - /* All pages of the b-tree have been visited. Return successfully. */ - *pnEntry = nEntry; - return SQLITE_OK; - } - moveToParent(pCur); - }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); - - pCur->aiIdx[pCur->iPage]++; - pPage = pCur->apPage[pCur->iPage]; + rc = moveToRightmost(pCur); + }else{ + while( pCur->aiIdx[pCur->iPage]==0 ){ + if( pCur->iPage==0 ){ + pCur->eState = CURSOR_INVALID; + *pRes = 1; + return SQLITE_OK; + } + moveToParent(pCur); } + pCur->info.nSize = 0; + pCur->validNKey = 0; - /* Descend to the child node of the cell that the cursor currently - ** points at. This is the right-child if (iIdx==pPage->nCell). - */ - iIdx = pCur->aiIdx[pCur->iPage]; - if( iIdx==pPage->nCell ){ - rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + pCur->aiIdx[pCur->iPage]--; + pPage = pCur->apPage[pCur->iPage]; + if( pPage->intKey && !pPage->leaf ){ + rc = sqlite3BtreePrevious(pCur, pRes); }else{ - rc = moveToChild(pCur, get4byte(findCell(pPage, iIdx))); + rc = SQLITE_OK; } } - - /* An error has occurred. Return an error code. */ + *pRes = 0; return rc; } -#endif - -/* -** Return the pager associated with a BTree. This routine is used for -** testing and debugging only. -*/ -SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ - return p->pBt->pPager; -} - -#ifndef SQLITE_OMIT_INTEGRITY_CHECK -/* -** Append a message to the error message string. -*/ -static void checkAppendMsg( - IntegrityCk *pCheck, - char *zMsg1, - const char *zFormat, - ... -){ - va_list ap; - if( !pCheck->mxErr ) return; - pCheck->mxErr--; - pCheck->nErr++; - va_start(ap, zFormat); - if( pCheck->errMsg.nChar ){ - sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); - } - if( zMsg1 ){ - sqlite3StrAccumAppend(&pCheck->errMsg, zMsg1, -1); - } - sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); - va_end(ap); - if( pCheck->errMsg.mallocFailed ){ - pCheck->mallocFailed = 1; - } -} -#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ -#ifndef SQLITE_OMIT_INTEGRITY_CHECK /* -** Add 1 to the reference count for page iPage. If this is the second -** reference to the page, add an error message to pCheck->zErrMsg. -** Return 1 if there are 2 ore more references to the page and 0 if -** if this is the first reference to the page. +** Allocate a new page from the database file. ** -** Also check that the page number is in bounds. -*/ -static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ - if( iPage==0 ) return 1; - if( iPage>pCheck->nPage ){ - checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); - return 1; - } - if( pCheck->anRef[iPage]==1 ){ - checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); - return 1; - } - return (pCheck->anRef[iPage]++)>1; -} - -#ifndef SQLITE_OMIT_AUTOVACUUM -/* -** Check that the entry in the pointer-map for page iChild maps to -** page iParent, pointer type ptrType. If not, append an error message -** to pCheck. +** The new page is marked as dirty. (In other words, sqlite3PagerWrite() +** has already been called on the new page.) The new page has also +** been referenced and the calling routine is responsible for calling +** sqlite3PagerUnref() on the new page when it is done. +** +** SQLITE_OK is returned on success. Any other return value indicates +** an error. *ppPage and *pPgno are undefined in the event of an error. +** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. +** +** If the "nearby" parameter is not 0, then a (feeble) effort is made to +** locate a page close to the page number "nearby". This can be used in an +** attempt to keep related pages close to each other in the database file, +** which in turn can make database access faster. +** +** If the "exact" parameter is not 0, and the page-number nearby exists +** anywhere on the free-list, then it is guarenteed to be returned. This +** is only used by auto-vacuum databases when allocating a new table. */ -static void checkPtrmap( - IntegrityCk *pCheck, /* Integrity check context */ - Pgno iChild, /* Child page number */ - u8 eType, /* Expected pointer map type */ - Pgno iParent, /* Expected pointer map parent page number */ - char *zContext /* Context description (used for error msg) */ +static int allocateBtreePage( + BtShared *pBt, + MemPage **ppPage, + Pgno *pPgno, + Pgno nearby, + u8 exact ){ + MemPage *pPage1; int rc; - u8 ePtrmapType; - Pgno iPtrmapParent; - - rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; - checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); - return; - } + u32 n; /* Number of pages on the freelist */ + u32 k; /* Number of leaves on the trunk of the freelist */ + MemPage *pTrunk = 0; + MemPage *pPrevTrunk = 0; + Pgno mxPage; /* Total size of the database file */ - if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ - checkAppendMsg(pCheck, zContext, - "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", - iChild, eType, iParent, ePtrmapType, iPtrmapParent); + assert( sqlite3_mutex_held(pBt->mutex) ); + pPage1 = pBt->pPage1; + mxPage = btreePagecount(pBt); + n = get4byte(&pPage1->aData[36]); + testcase( n==mxPage-1 ); + if( n>=mxPage ){ + return SQLITE_CORRUPT_BKPT; } -} -#endif - -/* -** Check the integrity of the freelist or of an overflow page list. -** Verify that the number of pages on the list is N. -*/ -static void checkList( - IntegrityCk *pCheck, /* Integrity checking context */ - int isFreeList, /* True for a freelist. False for overflow page list */ - int iPage, /* Page number for first page in the list */ - int N, /* Expected number of pages in the list */ - char *zContext /* Context for error messages */ -){ - int i; - int expected = N; - int iFirst = iPage; - while( N-- > 0 && pCheck->mxErr ){ - DbPage *pOvflPage; - unsigned char *pOvflData; - if( iPage<1 ){ - checkAppendMsg(pCheck, zContext, - "%d of %d pages missing from overflow list starting at %d", - N+1, expected, iFirst); - break; - } - if( checkRef(pCheck, iPage, zContext) ) break; - if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ - checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); - break; - } - pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); - if( isFreeList ){ - int n = get4byte(&pOvflData[4]); + if( n>0 ){ + /* There are pages on the freelist. Reuse one of those pages. */ + Pgno iTrunk; + u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + + /* If the 'exact' parameter was true and a query of the pointer-map + ** shows that the page 'nearby' is somewhere on the free-list, then + ** the entire-list will be searched for that page. + */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + if( exact && nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; } + *pPgno = nearby; + } #endif - if( n>(int)pCheck->pBt->usableSize/4-2 ){ - checkAppendMsg(pCheck, zContext, - "freelist leaf count too big on page %d", iPage); - N--; + + /* Decrement the free-list count by 1. Set iTrunk to the index of the + ** first free-list trunk page. iPrevTrunk is initially 1. + */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) return rc; + put4byte(&pPage1->aData[36], n-1); + + /* The code within this loop is run only once if the 'searchList' variable + ** is not true. Otherwise, it runs once for each trunk-page on the + ** free-list until the page 'nearby' is located. + */ + do { + pPrevTrunk = pTrunk; + if( pPrevTrunk ){ + iTrunk = get4byte(&pPrevTrunk->aData[0]); }else{ - for(i=0; iaData[32]); + } + testcase( iTrunk==mxPage ); + if( iTrunk>mxPage ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + } + if( rc ){ + pTrunk = 0; + goto end_allocate_page; + } + + k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + if( k==0 && !searchList ){ + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly + ** allocated page */ + assert( pPrevTrunk==0 ); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + *pPgno = iTrunk; + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + *ppPage = pTrunk; + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + }else if( k>(u32)(pBt->usableSize/4 - 2) ){ + /* Value of k is out of range. Database corruption */ + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + }else if( searchList && nearby==iTrunk ){ + /* The list is being searched and this trunk page is the page + ** to allocate, regardless of whether it has leaves. + */ + assert( *pPgno==iTrunk ); + *ppPage = pTrunk; + searchList = 0; + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + if( k==0 ){ + if( !pPrevTrunk ){ + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); + } + }else{ + /* The trunk page is required by the caller but it contains + ** pointers to free-list leaves. The first leaf becomes a trunk + ** page in this case. + */ + MemPage *pNewTrunk; + Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); + if( iNewTrunk>mxPage ){ + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; + } + testcase( iNewTrunk==mxPage ); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + rc = sqlite3PagerWrite(pNewTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pNewTrunk); + goto end_allocate_page; + } + memcpy(&pNewTrunk->aData[0], &pTrunk->aData[0], 4); + put4byte(&pNewTrunk->aData[4], k-1); + memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); + releasePage(pNewTrunk); + if( !pPrevTrunk ){ + assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); + put4byte(&pPage1->aData[32], iNewTrunk); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + put4byte(&pPrevTrunk->aData[0], iNewTrunk); } + } + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); #endif - checkRef(pCheck, iFreePage, zContext); + }else if( k>0 ){ + /* Extract a leaf from the trunk */ + u32 closest; + Pgno iPage; + unsigned char *aData = pTrunk->aData; + if( nearby>0 ){ + u32 i; + int dist; + closest = 0; + dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby); + for(i=1; imxPage ){ + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; + } + testcase( iPage==mxPage ); + if( !searchList || iPage==nearby ){ + int noContent; + *pPgno = iPage; + TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" + ": %d more free pages\n", + *pPgno, closest+1, k, pTrunk->pgno, n-1)); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ) goto end_allocate_page; + if( closestpDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } + } + searchList = 0; } - N -= n; } - } + releasePage(pPrevTrunk); + pPrevTrunk = 0; + }while( searchList ); + }else{ + /* There are no pages on the freelist, so create a new page at the + ** end of the file */ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; + #ifndef SQLITE_OMIT_AUTOVACUUM - else{ - /* If this database supports auto-vacuum and iPage is not the last - ** page in this overflow list, check that the pointer-map entry for - ** the following page matches iPage. + if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, pBt->nPage) ){ + /* If *pPgno refers to a pointer-map page, allocate two new pages + ** at the end of the file instead of one. The first allocated page + ** becomes a new pointer-map page, the second is used by the caller. */ - if( pCheck->pBt->autoVacuum && N>0 ){ - i = get4byte(pOvflData); - checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + MemPage *pPg = 0; + TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg->pDbPage); + releasePage(pPg); } + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ){ pBt->nPage++; } } #endif - iPage = get4byte(pOvflData); - sqlite3PagerUnref(pOvflPage); + put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage); + *pPgno = pBt->nPage; + + assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetPage(pBt, *pPgno, ppPage, 1); + if( rc ) return rc; + rc = sqlite3PagerWrite((*ppPage)->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + } + TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + } + + assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + +end_allocate_page: + releasePage(pTrunk); + releasePage(pPrevTrunk); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; } + assert( rc!=SQLITE_OK || sqlite3PagerIswriteable((*ppPage)->pDbPage) ); + return rc; } -#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ -#ifndef SQLITE_OMIT_INTEGRITY_CHECK /* -** Do various sanity checks on a single page of a tree. Return -** the tree depth. Root pages return 0. Parents of root pages -** return 1, and so forth. -** -** These checks are done: +** This function is used to add page iPage to the database file free-list. +** It is assumed that the page is not already a part of the free-list. ** -** 1. Make sure that cells and freeblocks do not overlap -** but combine to completely cover the page. -** NO 2. Make sure cell keys are in order. -** NO 3. Make sure no key is less than or equal to zLowerBound. -** NO 4. Make sure no key is greater than or equal to zUpperBound. -** 5. Check the integrity of overflow pages. -** 6. Recursively call checkTreePage on all children. -** 7. Verify that the depth of all children is the same. -** 8. Make sure this page is at least 33% full or else it is -** the root of the tree. +** The value passed as the second argument to this function is optional. +** If the caller happens to have a pointer to the MemPage object +** corresponding to page iPage handy, it may pass it as the second value. +** Otherwise, it may pass NULL. +** +** If a pointer to a MemPage object is passed as the second argument, +** its reference count is not altered by this function. */ -static int checkTreePage( - IntegrityCk *pCheck, /* Context for the sanity check */ - int iPage, /* Page number of the page to check */ - char *zParentContext, /* Parent context */ - i64 *pnParentMinKey, - i64 *pnParentMaxKey -){ - MemPage *pPage; - int i, rc, depth, d2, pgno, cnt; - int hdr, cellStart; - int nCell; - u8 *data; - BtShared *pBt; - int usableSize; - char zContext[100]; - char *hit = 0; - i64 nMinKey = 0; - i64 nMaxKey = 0; +static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ + MemPage *pTrunk = 0; /* Free-list trunk page */ + Pgno iTrunk = 0; /* Page number of free-list trunk page */ + MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ + MemPage *pPage; /* Page being freed. May be NULL. */ + int rc; /* Return Code */ + int nFree; /* Initial number of pages on free-list */ - sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( iPage>1 ); + assert( !pMemPage || pMemPage->pgno==iPage ); - /* Check that the page exists - */ - pBt = pCheck->pBt; - usableSize = pBt->usableSize; - if( iPage==0 ) return 0; - if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ - checkAppendMsg(pCheck, zContext, - "unable to get the page. error code=%d", rc); - return 0; - } - - /* Clear MemPage.isInit to make sure the corruption detection code in - ** btreeInitPage() is executed. */ - pPage->isInit = 0; - if( (rc = btreeInitPage(pPage))!=0 ){ - assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ - checkAppendMsg(pCheck, zContext, - "btreeInitPage() returns error code %d", rc); - releasePage(pPage); - return 0; + if( pMemPage ){ + pPage = pMemPage; + sqlite3PagerRef(pPage->pDbPage); + }else{ + pPage = btreePageLookup(pBt, iPage); } - /* Check out all the cells. - */ - depth = 0; - for(i=0; inCell && pCheck->mxErr; i++){ - u8 *pCell; - u32 sz; - CellInfo info; + /* Increment the free page count on pPage1 */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) goto freepage_out; + nFree = get4byte(&pPage1->aData[36]); + put4byte(&pPage1->aData[36], nFree+1); - /* Check payload overflow pages - */ - sqlite3_snprintf(sizeof(zContext), zContext, - "On tree page %d cell %d: ", iPage, i); - pCell = findCell(pPage,i); - btreeParseCellPtr(pPage, pCell, &info); - sz = info.nData; - if( !pPage->intKey ) sz += (int)info.nKey; - /* For intKey pages, check that the keys are in order. + if( pBt->secureDelete ){ + /* If the secure_delete option is enabled, then + ** always fully overwrite deleted information with zeros. */ - else if( i==0 ) nMinKey = nMaxKey = info.nKey; - else{ - if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); - } - nMaxKey = info.nKey; - } - assert( sz==info.nPayload ); - if( (sz>info.nLocal) - && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ - int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); - Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); - } -#endif - checkList(pCheck, 0, pgnoOvfl, nPage, zContext); - } - - /* Check sanity of left child page. - */ - if( !pPage->leaf ){ - pgno = get4byte(pCell); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); - } -#endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); - if( i>0 && d2!=depth ){ - checkAppendMsg(pCheck, zContext, "Child page depth differs"); - } - depth = d2; - } - } - - if( !pPage->leaf ){ - pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - sqlite3_snprintf(sizeof(zContext), zContext, - "On page %d at right child: ", iPage); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); - } -#endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); - } - - /* For intKey leaf pages, check that the min/max keys are in order - ** with any left/parent/right pages. - */ - if( pPage->leaf && pPage->intKey ){ - /* if we are a left child page */ - if( pnParentMinKey ){ - /* if we are the left most child page */ - if( !pnParentMaxKey ){ - if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (max larger than parent min of %lld)", - nMaxKey, *pnParentMinKey); - } - }else{ - if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (min less than parent min of %lld)", - nMinKey, *pnParentMinKey); - } - if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (max larger than parent max of %lld)", - nMaxKey, *pnParentMaxKey); - } - *pnParentMinKey = nMaxKey; - } - /* else if we're a right child page */ - } else if( pnParentMaxKey ){ - if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (min less than parent max of %lld)", - nMinKey, *pnParentMaxKey); - } + goto freepage_out; } + memset(pPage->aData, 0, pPage->pBt->pageSize); } - /* Check for complete coverage of the page + /* If the database supports auto-vacuum, write an entry in the pointer-map + ** to indicate that the page is free. */ - data = pPage->aData; - hdr = pPage->hdrOffset; - hit = sqlite3PageMalloc( pBt->pageSize ); - if( hit==0 ){ - pCheck->mallocFailed = 1; - }else{ - int contentOffset = get2byteNotZero(&data[hdr+5]); - assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ - memset(hit+contentOffset, 0, usableSize-contentOffset); - memset(hit, 1, contentOffset); - nCell = get2byte(&data[hdr+3]); - cellStart = hdr + 12 - 4*pPage->leaf; - for(i=0; i=usableSize ){ - checkAppendMsg(pCheck, 0, - "Corruption detected in cell %d on page %d",i,iPage); - }else{ - for(j=pc+size-1; j>=pc; j--) hit[j]++; - } - } - i = get2byte(&data[hdr+1]); - while( i>0 ){ - int size, j; - assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ - size = get2byte(&data[i+2]); - assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ - for(j=i+size-1; j>=i; j--) hit[j]++; - j = get2byte(&data[i]); - assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ - i = j; - } - for(i=cnt=0; i1 ){ - checkAppendMsg(pCheck, 0, - "Multiple uses for byte %d of page %d", i, iPage); - break; - } - } - if( cnt!=data[hdr+7] ){ - checkAppendMsg(pCheck, 0, - "Fragmentation of %d bytes reported as %d on page %d", - cnt, data[hdr+7], iPage); - } - } - sqlite3PageFree(hit); - releasePage(pPage); - return depth+1; -} -#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ - -#ifndef SQLITE_OMIT_INTEGRITY_CHECK -/* -** This routine does a complete check of the given BTree file. aRoot[] is -** an array of pages numbers were each page number is the root page of -** a table. nRoot is the number of entries in aRoot. -** -** A read-only or read-write transaction must be opened before calling -** this function. -** -** Write the number of error seen in *pnErr. Except for some memory -** allocation errors, an error message held in memory obtained from -** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is -** returned. If a memory allocation error occurs, NULL is returned. -*/ -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( - Btree *p, /* The btree to be checked */ - int *aRoot, /* An array of root pages numbers for individual trees */ - int nRoot, /* Number of entries in aRoot[] */ - int mxErr, /* Stop reporting errors after this many */ - int *pnErr /* Write number of errors seen to this variable */ -){ - Pgno i; - int nRef; - IntegrityCk sCheck; - BtShared *pBt = p->pBt; - char zErr[100]; - - sqlite3BtreeEnter(p); - assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - nRef = sqlite3PagerRefcount(pBt->pPager); - sCheck.pBt = pBt; - sCheck.pPager = pBt->pPager; - sCheck.nPage = btreePagecount(sCheck.pBt); - sCheck.mxErr = mxErr; - sCheck.nErr = 0; - sCheck.mallocFailed = 0; - *pnErr = 0; - if( sCheck.nPage==0 ){ - sqlite3BtreeLeave(p); - return 0; - } - sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) ); - if( !sCheck.anRef ){ - *pnErr = 1; - sqlite3BtreeLeave(p); - return 0; - } - for(i=0; i<=sCheck.nPage; i++){ sCheck.anRef[i] = 0; } - i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ){ - sCheck.anRef[i] = 1; + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); + if( rc ) goto freepage_out; } - sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); - sCheck.errMsg.useMalloc = 2; - /* Check the integrity of the freelist + /* Now manipulate the actual database free-list structure. There are two + ** possibilities. If the free-list is currently empty, or if the first + ** trunk page in the free-list is full, then this page will become a + ** new free-list trunk page. Otherwise, it will become a leaf of the + ** first trunk page in the current free-list. This block tests if it + ** is possible to add the page as a new free-list leaf. */ - checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), - get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); + if( nFree!=0 ){ + u32 nLeaf; /* Initial number of leaf cells on trunk page */ - /* Check all the tables. - */ - for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + iTrunk = get4byte(&pPage1->aData[32]); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + if( rc!=SQLITE_OK ){ + goto freepage_out; } -#endif - checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); - } - /* Make sure every page in the file is referenced - */ - for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ -#ifdef SQLITE_OMIT_AUTOVACUUM - if( sCheck.anRef[i]==0 ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); - } -#else - /* If the database supports auto-vacuum, make sure no tables contain - ** references to pointer-map pages. - */ - if( sCheck.anRef[i]==0 && - (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + nLeaf = get4byte(&pTrunk->aData[4]); + assert( pBt->usableSize>32 ); + if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; } - if( sCheck.anRef[i]!=0 && - (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ + /* In this case there is room on the trunk page to insert the page + ** being freed as a new leaf. + ** + ** Note that the trunk page is not really full until it contains + ** usableSize/4 - 2 entries, not usableSize/4 - 8 entries as we have + ** coded. But due to a coding error in versions of SQLite prior to + ** 3.6.0, databases with freelist trunk pages holding more than + ** usableSize/4 - 8 entries will be reported as corrupt. In order + ** to maintain backwards compatibility with older versions of SQLite, + ** we will continue to restrict the number of entries to usableSize/4 - 8 + ** for now. At some point in the future (once everyone has upgraded + ** to 3.6.0 or later) we should consider fixing the conditional above + ** to read "usableSize/4-2" instead of "usableSize/4-8". + */ + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pTrunk->aData[4], nLeaf+1); + put4byte(&pTrunk->aData[8+nLeaf*4], iPage); + if( pPage && !pBt->secureDelete ){ + sqlite3PagerDontWrite(pPage->pDbPage); + } + rc = btreeSetHasContent(pBt, iPage); + } + TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + goto freepage_out; } -#endif } - /* Make sure this analysis did not leave any unref() pages. - ** This is an internal consistency check; an integrity check - ** of the integrity check. + /* If control flows to this point, then it was not possible to add the + ** the page being freed as a leaf page of the first trunk in the free-list. + ** Possibly because the free-list is empty, or possibly because the + ** first trunk in the free-list is full. Either way, the page being freed + ** will become the new first trunk page in the free-list. */ - if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, 0, - "Outstanding page count goes from %d to %d during this analysis", - nRef, sqlite3PagerRefcount(pBt->pPager) - ); + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + goto freepage_out; + } + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + goto freepage_out; } + put4byte(pPage->aData, iTrunk); + put4byte(&pPage->aData[4], 0); + put4byte(&pPage1->aData[32], iPage); + TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); - /* Clean up and report errors. - */ - sqlite3BtreeLeave(p); - sqlite3_free(sCheck.anRef); - if( sCheck.mallocFailed ){ - sqlite3StrAccumReset(&sCheck.errMsg); - *pnErr = sCheck.nErr+1; - return 0; +freepage_out: + if( pPage ){ + pPage->isInit = 0; } - *pnErr = sCheck.nErr; - if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); - return sqlite3StrAccumFinish(&sCheck.errMsg); + releasePage(pPage); + releasePage(pTrunk); + return rc; } -#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ - -/* -** Return the full pathname of the underlying database file. -** -** The pager filename is invariant as long as the pager is -** open so it is safe to access without the BtShared mutex. -*/ -SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *p){ - assert( p->pBt->pPager!=0 ); - return sqlite3PagerFilename(p->pBt->pPager); +static void freePage(MemPage *pPage, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + *pRC = freePage2(pPage->pBt, pPage, pPage->pgno); + } } /* -** Return the pathname of the journal file for this database. The return -** value of this routine is the same regardless of whether the journal file -** has been created or not. -** -** The pager journal filename is invariant as long as the pager is -** open so it is safe to access without the BtShared mutex. +** Free any overflow pages associated with the given Cell. */ -SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *p){ - assert( p->pBt->pPager!=0 ); - return sqlite3PagerJournalname(p->pBt->pPager); -} +static int clearCell(MemPage *pPage, unsigned char *pCell){ + BtShared *pBt = pPage->pBt; + CellInfo info; + Pgno ovflPgno; + int rc; + int nOvfl; + u32 ovflPageSize; -/* -** Return non-zero if a transaction is active. -*/ -SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree *p){ - assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); - return (p && (p->inTrans==TRANS_WRITE)); -} + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + btreeParseCellPtr(pPage, pCell, &info); + if( info.iOverflow==0 ){ + return SQLITE_OK; /* No overflow pages. Return without doing anything */ + } + ovflPgno = get4byte(&pCell[info.iOverflow]); + assert( pBt->usableSize > 4 ); + ovflPageSize = pBt->usableSize - 4; + nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; + assert( ovflPgno==0 || nOvfl>0 ); + while( nOvfl-- ){ + Pgno iNext = 0; + MemPage *pOvfl = 0; + if( ovflPgno<2 || ovflPgno>btreePagecount(pBt) ){ + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the + ** file the database must be corrupt. */ + return SQLITE_CORRUPT_BKPT; + } + if( nOvfl ){ + rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); + if( rc ) return rc; + } -#ifndef SQLITE_OMIT_WAL -/* -** Run a checkpoint on the Btree passed as the first argument. -** -** Return SQLITE_LOCKED if this or any other connection has an open -** transaction on the shared-cache the argument Btree is connected to. -*/ -SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p){ - int rc = SQLITE_OK; - if( p ){ - BtShared *pBt = p->pBt; - sqlite3BtreeEnter(p); - if( pBt->inTransaction!=TRANS_NONE ){ - rc = SQLITE_LOCKED; + if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) + && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 + ){ + /* There is no reason any cursor should have an outstanding reference + ** to an overflow page belonging to a cell that is being deleted/updated. + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as + ** freePage2() may zero the page contents if secure-delete mode is + ** enabled. If this 'overflow' page happens to be a page that the + ** caller is iterating through or using in some other way, this + ** can be problematic. + */ + rc = SQLITE_CORRUPT_BKPT; }else{ - rc = sqlite3PagerCheckpoint(pBt->pPager); + rc = freePage2(pBt, pOvfl, ovflPgno); } - sqlite3BtreeLeave(p); + + if( pOvfl ){ + sqlite3PagerUnref(pOvfl->pDbPage); + } + if( rc ) return rc; + ovflPgno = iNext; } - return rc; + return SQLITE_OK; } -#endif /* -** Return non-zero if a read (or write) transaction is active. +** Create the byte sequence used to represent a cell on page pPage +** and write that byte sequence into pCell[]. Overflow pages are +** allocated and filled in as necessary. The calling procedure +** is responsible for making sure sufficient space has been allocated +** for pCell[]. +** +** Note that pCell does not necessary need to point to the pPage->aData +** area. pCell might point to some temporary storage. The cell will +** be constructed in this temporary area then copied into pPage->aData +** later. */ -SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree *p){ - assert( p ); - assert( sqlite3_mutex_held(p->db->mutex) ); - return p->inTrans!=TRANS_NONE; -} +static int fillInCell( + MemPage *pPage, /* The page that contains the cell */ + unsigned char *pCell, /* Complete text of the cell */ + const void *pKey, i64 nKey, /* The key */ + const void *pData,int nData, /* The data */ + int nZero, /* Extra zero bytes to append to pData */ + int *pnSize /* Write cell size here */ +){ + int nPayload; + const u8 *pSrc; + int nSrc, n, rc; + int spaceLeft; + MemPage *pOvfl = 0; + MemPage *pToRelease = 0; + unsigned char *pPrior; + unsigned char *pPayload; + BtShared *pBt = pPage->pBt; + Pgno pgnoOvfl = 0; + int nHeader; + CellInfo info; -SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ - assert( p ); - assert( sqlite3_mutex_held(p->db->mutex) ); - return p->nBackup!=0; -} + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); -/* -** This function returns a pointer to a blob of memory associated with -** a single shared-btree. The memory is used by client code for its own -** purposes (for example, to store a high-level schema associated with -** the shared-btree). The btree layer manages reference counting issues. -** -** The first time this is called on a shared-btree, nBytes bytes of memory -** are allocated, zeroed, and returned to the caller. For each subsequent -** call the nBytes parameter is ignored and a pointer to the same blob -** of memory returned. -** -** If the nBytes parameter is 0 and the blob of memory has not yet been -** allocated, a null pointer is returned. If the blob has already been -** allocated, it is returned as normal. -** -** Just before the shared-btree is closed, the function passed as the -** xFree argument when the memory allocation was made is invoked on the -** blob of allocated memory. This function should not call sqlite3_free() -** on the memory, the btree layer does that. -*/ -SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ - BtShared *pBt = p->pBt; - sqlite3BtreeEnter(p); - if( !pBt->pSchema && nBytes ){ - pBt->pSchema = sqlite3DbMallocZero(0, nBytes); - pBt->xFreeSchema = xFree; + /* pPage is not necessarily writeable since pCell might be auxiliary + ** buffer space that is separate from the pPage buffer area */ + assert( pCellaData || pCell>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + /* Fill in the header. */ + nHeader = 0; + if( !pPage->leaf ){ + nHeader += 4; } - sqlite3BtreeLeave(p); - return pBt->pSchema; -} + if( pPage->hasData ){ + nHeader += putVarint(&pCell[nHeader], nData+nZero); + }else{ + nData = nZero = 0; + } + nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); + btreeParseCellPtr(pPage, pCell, &info); + assert( info.nHeader==nHeader ); + assert( info.nKey==nKey ); + assert( info.nData==(u32)(nData+nZero) ); + + /* Fill in the payload */ + nPayload = nData + nZero; + if( pPage->intKey ){ + pSrc = pData; + nSrc = nData; + nData = 0; + }else{ + if( NEVER(nKey>0x7fffffff || pKey==0) ){ + return SQLITE_CORRUPT_BKPT; + } + nPayload += (int)nKey; + pSrc = pKey; + nSrc = (int)nKey; + } + *pnSize = info.nSize; + spaceLeft = info.nLocal; + pPayload = &pCell[nHeader]; + pPrior = &pCell[info.iOverflow]; -/* -** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared -** btree as the argument handle holds an exclusive lock on the -** sqlite_master table. Otherwise SQLITE_OK. -*/ -SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ - int rc; - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); - assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); - sqlite3BtreeLeave(p); - return rc; -} + while( nPayload>0 ){ + if( spaceLeft==0 ){ +#ifndef SQLITE_OMIT_AUTOVACUUM + Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ + if( pBt->autoVacuum ){ + do{ + pgnoOvfl++; + } while( + PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) + ); + } +#endif + rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, and the second or subsequent + ** overflow page is being allocated, add an entry to the pointer-map + ** for that page now. + ** + ** If this is the first overflow page, then write a partial entry + ** to the pointer-map. If we write nothing to this pointer-map slot, + ** then the optimistic overflow chain processing in clearCell() + ** may misinterpret the uninitialised values and delete the + ** wrong pages from the database. + */ + if( pBt->autoVacuum && rc==SQLITE_OK ){ + u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); + ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); + if( rc ){ + releasePage(pOvfl); + } + } +#endif + if( rc ){ + releasePage(pToRelease); + return rc; + } + /* If pToRelease is not zero than pPrior points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); -#ifndef SQLITE_OMIT_SHARED_CACHE -/* -** Obtain a lock on the table whose root page is iTab. The -** lock is a write lock if isWritelock is true or a read lock -** if it is false. -*/ -SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ - int rc = SQLITE_OK; - assert( p->inTrans!=TRANS_NONE ); - if( p->sharable ){ - u8 lockType = READ_LOCK + isWriteLock; - assert( READ_LOCK+1==WRITE_LOCK ); - assert( isWriteLock==0 || isWriteLock==1 ); + /* If pPrior is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPrioraData || pPrior>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); - sqlite3BtreeEnter(p); - rc = querySharedCacheTableLock(p, iTab, lockType); - if( rc==SQLITE_OK ){ - rc = setSharedCacheTableLock(p, iTab, lockType); + put4byte(pPrior, pgnoOvfl); + releasePage(pToRelease); + pToRelease = pOvfl; + pPrior = pOvfl->aData; + put4byte(pPrior, 0); + pPayload = &pOvfl->aData[4]; + spaceLeft = pBt->usableSize - 4; + } + n = nPayload; + if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + if( nSrc>0 ){ + if( n>nSrc ) n = nSrc; + assert( pSrc ); + memcpy(pPayload, pSrc, n); + }else{ + memset(pPayload, 0, n); + } + nPayload -= n; + pPayload += n; + pSrc += n; + nSrc -= n; + spaceLeft -= n; + if( nSrc==0 ){ + nSrc = nData; + pSrc = pData; } - sqlite3BtreeLeave(p); } - return rc; + releasePage(pToRelease); + return SQLITE_OK; } -#endif -#ifndef SQLITE_OMIT_INCRBLOB /* -** Argument pCsr must be a cursor opened for writing on an -** INTKEY table currently pointing at a valid table entry. -** This function modifies the data stored as part of that entry. +** Remove the i-th cell from pPage. This routine effects pPage only. +** The cell content is not freed or deallocated. It is assumed that +** the cell content has been copied someplace else. This routine just +** removes the reference to the cell from pPage. ** -** Only the data content may only be modified, it is not possible to -** change the length of the data stored. If this function is called with -** parameters that attempt to write past the end of the existing data, -** no modifications are made and SQLITE_CORRUPT is returned. +** "sz" must be the number of bytes in the cell. */ -SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ - int rc; - assert( cursorHoldsMutex(pCsr) ); - assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); - assert( pCsr->isIncrblobHandle ); +static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ + u32 pc; /* Offset to cell content of cell being deleted */ + u8 *data; /* pPage->aData */ + u8 *ptr; /* Used to move bytes around within data[] */ + u8 *endPtr; /* End of loop */ + int rc; /* The return code */ + int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ - rc = restoreCursorPosition(pCsr); - if( rc!=SQLITE_OK ){ - return rc; + if( *pRC ) return; + + assert( idx>=0 && idxnCell ); + assert( sz==cellSize(pPage, idx) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + data = pPage->aData; + ptr = &data[pPage->cellOffset + 2*idx]; + pc = get2byte(ptr); + hdr = pPage->hdrOffset; + testcase( pc==get2byte(&data[hdr+5]) ); + testcase( pc+sz==pPage->pBt->usableSize ); + if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; } - assert( pCsr->eState!=CURSOR_REQUIRESEEK ); - if( pCsr->eState!=CURSOR_VALID ){ - return SQLITE_ABORT; + rc = freeSpace(pPage, pc, sz); + if( rc ){ + *pRC = rc; + return; } - - /* Check some assumptions: - ** (a) the cursor is open for writing, - ** (b) there is a read/write transaction open, - ** (c) the connection holds a write-lock on the table (if required), - ** (d) there are no conflicting read-locks, and - ** (e) the cursor points at a valid row of an intKey table. - */ - if( !pCsr->wrFlag ){ - return SQLITE_READONLY; + endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptrpBt->readOnly && pCsr->pBt->inTransaction==TRANS_WRITE ); - assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); - assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); - assert( pCsr->apPage[pCsr->iPage]->intKey ); - - return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); + pPage->nCell--; + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; } -/* -** Set a flag on this cursor to cache the locations of pages from the -** overflow list for the current row. This is used by cursors opened -** for incremental blob IO only. +/* +** Insert a new cell on pPage at cell index "i". pCell points to the +** content of the cell. ** -** This function sets a flag only. The actual page location cache -** (stored in BtCursor.aOverflow[]) is allocated and used by function -** accessPayload() (the worker function for sqlite3BtreeData() and -** sqlite3BtreePutData()). +** If the cell content will fit on the page, then put it there. If it +** will not fit, then make a copy of the cell content into pTemp if +** pTemp is not null. Regardless of pTemp, allocate a new entry +** in pPage->aOvfl[] and make it point to the cell content (either +** in pTemp or the original pCell) and also record its index. +** Allocating a new entry in pPage->aCell[] implies that +** pPage->nOverflow is incremented. +** +** If nSkip is non-zero, then do not copy the first nSkip bytes of the +** cell. The caller will overwrite them after this function returns. If +** nSkip is non-zero, then pCell may not point to an invalid memory location +** (but pCell+nSkip is always valid). */ -SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - invalidateOverflowCache(pCur); - pCur->isIncrblobHandle = 1; -} +static void insertCell( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz, /* Bytes of content in pCell */ + u8 *pTemp, /* Temp storage space for pCell, if needed */ + Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ + int *pRC /* Read and write return code from here */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + int end; /* First byte past the last cell pointer in data[] */ + int ins; /* Index in data[] where new cell pointer is inserted */ + int cellOffset; /* Address of first cell pointer in data[] */ + u8 *data; /* The content of the whole page */ + u8 *ptr; /* Used for moving information around in data[] */ + u8 *endPtr; /* End of the loop */ + + int nSkip = (iChild ? 4 : 0); + + if( *pRC ) return; + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + /* The cell should normally be sized correctly. However, when moving a + ** malformed cell from a leaf page to an interior page, if the cell size + ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size + ** might be less than 8 (leaf-size + pointer) on the interior node. Hence + ** the term after the || in the following assert(). */ + assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); + if( pPage->nOverflow || sz+2>pPage->nFree ){ + if( pTemp ){ + memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); + pCell = pTemp; + } + if( iChild ){ + put4byte(pCell, iChild); + } + j = pPage->nOverflow++; + assert( j<(int)(sizeof(pPage->aOvfl)/sizeof(pPage->aOvfl[0])) ); + pPage->aOvfl[j].pCell = pCell; + pPage->aOvfl[j].idx = (u16)i; + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + cellOffset = pPage->cellOffset; + end = cellOffset + 2*pPage->nCell; + ins = cellOffset + 2*i; + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ *pRC = rc; return; } + /* The allocateSpace() routine guarantees the following two properties + ** if it returns success */ + assert( idx >= end+2 ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nCell++; + pPage->nFree -= (u16)(2 + sz); + memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); + if( iChild ){ + put4byte(&data[idx], iChild); + } + ptr = &data[end]; + endPtr = &data[ins]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptr>endPtr ){ + *(u16*)ptr = *(u16*)&ptr[-2]; + ptr -= 2; + } + put2byte(&data[ins], idx); + put2byte(&data[pPage->hdrOffset+3], pPage->nCell); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pCell, pRC); + } #endif + } +} /* -** Set both the "read version" (single byte at byte offset 18) and -** "write version" (single byte at byte offset 19) fields in the database -** header to iVersion. +** Add a list of cells to a page. The page should be initially empty. +** The cells are guaranteed to fit on the page. */ -SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ - BtShared *pBt = pBtree->pBt; - int rc; /* Return code */ - - assert( pBtree->inTrans==TRANS_NONE ); - assert( iVersion==1 || iVersion==2 ); +static void assemblePage( + MemPage *pPage, /* The page to be assemblied */ + int nCell, /* The number of cells to add to this page */ + u8 **apCell, /* Pointers to cell bodies */ + u16 *aSize /* Sizes of the cells */ +){ + int i; /* Loop counter */ + u8 *pCellptr; /* Address of next cell pointer */ + int cellbody; /* Address of next cell body */ + u8 * const data = pPage->aData; /* Pointer to data for pPage */ + const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ + const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ - /* If setting the version fields to 1, do not automatically open the - ** WAL connection, even if the version fields are currently set to 2. - */ - pBt->doNotUseWAL = (u8)(iVersion==1); + assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) + && (int)MX_CELL(pPage->pBt)<=10921); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - rc = sqlite3BtreeBeginTrans(pBtree, 0); - if( rc==SQLITE_OK ){ - u8 *aData = pBt->pPage1->aData; - if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ - rc = sqlite3BtreeBeginTrans(pBtree, 2); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - if( rc==SQLITE_OK ){ - aData[18] = (u8)iVersion; - aData[19] = (u8)iVersion; - } - } - } - } + /* Check that the page has just been zeroed by zeroPage() */ + assert( pPage->nCell==0 ); + assert( get2byteNotZero(&data[hdr+5])==nUsable ); - pBt->doNotUseWAL = 0; - return rc; + pCellptr = &data[pPage->cellOffset + nCell*2]; + cellbody = nUsable; + for(i=nCell-1; i>=0; i--){ + u16 sz = aSize[i]; + pCellptr -= 2; + cellbody -= sz; + put2byte(pCellptr, cellbody); + memcpy(&data[cellbody], apCell[i], sz); + } + put2byte(&data[hdr+3], nCell); + put2byte(&data[hdr+5], cellbody); + pPage->nFree -= (nCell*2 + nUsable - cellbody); + pPage->nCell = (u16)nCell; } -/************** End of btree.c ***********************************************/ -/************** Begin file backup.c ******************************************/ /* -** 2009 January 28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. ** -************************************************************************* -** This file contains the implementation of the sqlite3_backup_XXX() -** API functions and the related features. +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. */ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB (NN*2+1) /* Total pages involved in the balance */ -/* Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif +#ifndef SQLITE_OMIT_QUICKBALANCE /* -** Structure allocated for each backup operation. +** This version of balance() handles the common special case where +** a new entry is being inserted on the extreme right-end of the +** tree, in other words, when the new entry will become the largest +** entry in the tree. +** +** Instead of trying to balance the 3 right-most leaf pages, just add +** a new page to the right-hand side and put the one new entry in +** that page. This leaves the right side of the tree somewhat +** unbalanced. But odds are that we will be inserting new entries +** at the end soon afterwards so the nearly empty page will quickly +** fill up. On average. +** +** pPage is the leaf page which is the right-most page in the tree. +** pParent is its parent. pPage must have a single overflow entry +** which is also the right-most entry on the page. +** +** The pSpace buffer is used to store a temporary copy of the divider +** cell that will be inserted into pParent. Such a cell consists of a 4 +** byte page number followed by a variable length integer. In other +** words, at most 13 bytes. Hence the pSpace buffer must be at +** least 13 bytes in size. */ -struct sqlite3_backup { - sqlite3* pDestDb; /* Destination database handle */ - Btree *pDest; /* Destination b-tree file */ - u32 iDestSchema; /* Original schema cookie in destination */ - int bDestLocked; /* True once a write-transaction is open on pDest */ +static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ + BtShared *const pBt = pPage->pBt; /* B-Tree Database */ + MemPage *pNew; /* Newly allocated page */ + int rc; /* Return Code */ + Pgno pgnoNew; /* Page number of pNew */ - Pgno iNext; /* Page number of the next source page to copy */ - sqlite3* pSrcDb; /* Source database handle */ - Btree *pSrc; /* Source b-tree file */ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( pPage->nOverflow==1 ); - int rc; /* Backup process error code */ + /* This error condition is now caught prior to reaching this function */ + if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; - /* These two variables are set by every call to backup_step(). They are - ** read by calls to backup_remaining() and backup_pagecount(). + /* Allocate a new page. This page will become the right-sibling of + ** pPage. Make the parent page writable, so that the new divider cell + ** may be inserted. If both these operations are successful, proceed. */ - Pgno nRemaining; /* Number of pages left to copy */ - Pgno nPagecount; /* Total number of pages to copy */ + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); - int isAttached; /* True once backup has been registered with pager */ - sqlite3_backup *pNext; /* Next backup associated with source pager */ -}; + if( rc==SQLITE_OK ){ -/* -** THREAD SAFETY NOTES: -** -** Once it has been created using backup_init(), a single sqlite3_backup -** structure may be accessed via two groups of thread-safe entry points: -** -** * Via the sqlite3_backup_XXX() API function backup_step() and -** backup_finish(). Both these functions obtain the source database -** handle mutex and the mutex associated with the source BtShared -** structure, in that order. -** -** * Via the BackupUpdate() and BackupRestart() functions, which are -** invoked by the pager layer to report various state changes in -** the page cache associated with the source database. The mutex -** associated with the source database BtShared structure will always -** be held when either of these functions are invoked. -** -** The other sqlite3_backup_XXX() API functions, backup_remaining() and -** backup_pagecount() are not thread-safe functions. If they are called -** while some other thread is calling backup_step() or backup_finish(), -** the values returned may be invalid. There is no way for a call to -** BackupUpdate() or BackupRestart() to interfere with backup_remaining() -** or backup_pagecount(). -** -** Depending on the SQLite configuration, the database handles and/or -** the Btree objects may have their own mutexes that require locking. -** Non-sharable Btrees (in-memory databases for example), do not have -** associated mutexes. -*/ + u8 *pOut = &pSpace[4]; + u8 *pCell = pPage->aOvfl[0].pCell; + u16 szCell = cellSizePtr(pPage, pCell); + u8 *pStop; -/* -** Return a pointer corresponding to database zDb (i.e. "main", "temp") -** in connection handle pDb. If such a database cannot be found, return -** a NULL pointer and write an error message to pErrorDb. -** -** If the "temp" database is requested, it may need to be opened by this -** function. If an error occurs while doing so, return 0 and write an -** error message to pErrorDb. -*/ -static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ - int i = sqlite3FindDbName(pDb, zDb); + assert( sqlite3PagerIswriteable(pNew->pDbPage) ); + assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); + assemblePage(pNew, 1, &pCell, &szCell); - if( i==1 ){ - Parse *pParse; - int rc = 0; - pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); - if( pParse==0 ){ - sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory"); - rc = SQLITE_NOMEM; - }else{ - pParse->db = pDb; - if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); - rc = SQLITE_ERROR; + /* If this is an auto-vacuum database, update the pointer map + ** with entries for the new page, and any pointer from the + ** cell on the page to an overflow page. If either of these + ** operations fails, the return code is set, but the contents + ** of the parent page are still manipulated by thh code below. + ** That is Ok, at this point the parent page is guaranteed to + ** be marked as dirty. Returning an error code will cause a + ** rollback, undoing any changes made to the parent page. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); + if( szCell>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); } - sqlite3DbFree(pErrorDb, pParse->zErrMsg); - sqlite3StackFree(pErrorDb, pParse); - } - if( rc ){ - return 0; } - } + + /* Create a divider cell to insert into pParent. The divider cell + ** consists of a 4-byte page number (the page number of pPage) and + ** a variable length key value (which must be the same value as the + ** largest key on pPage). + ** + ** To find the largest key value on pPage, first find the right-most + ** cell on pPage. The first two fields of this cell are the + ** record-length (a variable length integer at most 32-bits in size) + ** and the key value (a variable length integer, may have any value). + ** The first of the while(...) loops below skips over the record-length + ** field. The second while(...) loop copies the key value from the + ** cell on pPage into the pSpace buffer. + */ + pCell = findCell(pPage, pPage->nCell-1); + pStop = &pCell[9]; + while( (*(pCell++)&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + + /* Set the right-child pointer of pParent to point to the new page. */ + put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); + + /* Release the reference to the new page. */ + releasePage(pNew); } - return pDb->aDb[i].pBt; + return rc; } +#endif /* SQLITE_OMIT_QUICKBALANCE */ +#if 0 /* -** Create an sqlite3_backup process to copy the contents of zSrcDb from -** connection handle pSrcDb to zDestDb in pDestDb. If successful, return -** a pointer to the new sqlite3_backup object. -** -** If an error occurs, NULL is returned and an error code and error message -** stored in database handle pDestDb. +** This function does not contribute anything to the operation of SQLite. +** it is sometimes activated temporarily while debugging code responsible +** for setting pointer-map entries. */ -SQLITE_API sqlite3_backup *sqlite3_backup_init( - sqlite3* pDestDb, /* Database to write to */ - const char *zDestDb, /* Name of database within pDestDb */ - sqlite3* pSrcDb, /* Database connection to read from */ - const char *zSrcDb /* Name of database within pSrcDb */ -){ - sqlite3_backup *p; /* Value to return */ - - /* Lock the source database handle. The destination database - ** handle is not locked in this routine, but it is locked in - ** sqlite3_backup_step(). The user is required to ensure that no - ** other thread accesses the destination handle for the duration - ** of the backup operation. Any attempt to use the destination - ** database connection while a backup is in progress may cause - ** a malfunction or a deadlock. - */ - sqlite3_mutex_enter(pSrcDb->mutex); - sqlite3_mutex_enter(pDestDb->mutex); +static int ptrmapCheckPages(MemPage **apPage, int nPage){ + int i, j; + for(i=0; ipBt; + assert( pPage->isInit ); - if( pSrcDb==pDestDb ){ - sqlite3Error( - pDestDb, SQLITE_ERROR, "source and destination must be distinct" - ); - p = 0; - }else { - /* Allocate space for a new sqlite3_backup object... - ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a - ** call to sqlite3_backup_init() and is destroyed by a call to - ** sqlite3_backup_finish(). */ - p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); - if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + for(j=0; jnCell; j++){ + CellInfo info; + u8 *z; + + z = findCell(pPage, j); + btreeParseCellPtr(pPage, z, &info); + if( info.iOverflow ){ + Pgno ovfl = get4byte(&z[info.iOverflow]); + ptrmapGet(pBt, ovfl, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_OVERFLOW1 ); + } + if( !pPage->leaf ){ + Pgno child = get4byte(z); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } } - } - - /* If the allocation succeeded, populate the new object. */ - if( p ){ - memset(p, 0, sizeof(sqlite3_backup)); - p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); - p->pDest = findBtree(pDestDb, pDestDb, zDestDb); - p->pDestDb = pDestDb; - p->pSrcDb = pSrcDb; - p->iNext = 1; - p->isAttached = 0; - - if( 0==p->pSrc || 0==p->pDest ){ - /* One (or both) of the named databases did not exist. An error has - ** already been written into the pDestDb handle. All that is left - ** to do here is free the sqlite3_backup structure. - */ - sqlite3_free(p); - p = 0; + if( !pPage->leaf ){ + Pgno child = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); } } - if( p ){ - p->pSrc->nBackup++; - } - - sqlite3_mutex_leave(pDestDb->mutex); - sqlite3_mutex_leave(pSrcDb->mutex); - return p; + return 1; } +#endif /* -** Argument rc is an SQLite error code. Return true if this error is -** considered fatal if encountered during a backup operation. All errors -** are considered fatal except for SQLITE_BUSY and SQLITE_LOCKED. +** This function is used to copy the contents of the b-tree node stored +** on page pFrom to page pTo. If page pFrom was not a leaf page, then +** the pointer-map entries for each child page are updated so that the +** parent page stored in the pointer map is page pTo. If pFrom contained +** any cells with overflow page pointers, then the corresponding pointer +** map entries are also updated so that the parent page is page pTo. +** +** If pFrom is currently carrying any overflow cells (entries in the +** MemPage.aOvfl[] array), they are not copied to pTo. +** +** Before returning, page pTo is reinitialized using btreeInitPage(). +** +** The performance of this function is not critical. It is only used by +** the balance_shallower() and balance_deeper() procedures, neither of +** which are called often under normal circumstances. */ -static int isFatalError(int rc){ - return (rc!=SQLITE_OK && rc!=SQLITE_BUSY && ALWAYS(rc!=SQLITE_LOCKED)); +static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + BtShared * const pBt = pFrom->pBt; + u8 * const aFrom = pFrom->aData; + u8 * const aTo = pTo->aData; + int const iFromHdr = pFrom->hdrOffset; + int const iToHdr = ((pTo->pgno==1) ? 100 : 0); + int rc; + int iData; + + + assert( pFrom->isInit ); + assert( pFrom->nFree>=iToHdr ); + assert( get2byte(&aFrom[iFromHdr+5]) <= (int)pBt->usableSize ); + + /* Copy the b-tree node content from page pFrom to page pTo. */ + iData = get2byte(&aFrom[iFromHdr+5]); + memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); + memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); + + /* Reinitialize page pTo so that the contents of the MemPage structure + ** match the new data. The initialization of pTo can actually fail under + ** fairly obscure circumstances, even though it is a copy of initialized + ** page pFrom. + */ + pTo->isInit = 0; + rc = btreeInitPage(pTo); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + + /* If this is an auto-vacuum database, update the pointer-map entries + ** for any b-tree or overflow pages that pTo now contains the pointers to. + */ + if( ISAUTOVACUUM ){ + *pRC = setChildPtrmaps(pTo); + } + } } /* -** Parameter zSrcData points to a buffer containing the data for -** page iSrcPg from the source database. Copy this data into the -** destination database. +** This routine redistributes cells on the iParentIdx'th child of pParent +** (hereafter "the page") and up to 2 siblings so that all pages have about the +** same amount of free space. Usually a single sibling on either side of the +** page are used in the balancing, though both siblings might come from one +** side if the page is the first or last child of its parent. If the page +** has fewer than 2 siblings (something which can only happen if the page +** is a root page or a child of a root page) then all available siblings +** participate in the balancing. +** +** The number of siblings of the page might be increased or decreased by +** one or two in an effort to keep pages nearly full but not over full. +** +** Note that when this routine is called, some of the cells on the page +** might not actually be stored in MemPage.aData[]. This can happen +** if the page is overfull. This routine ensures that all cells allocated +** to the page and its siblings fit into MemPage.aData[] before returning. +** +** In the course of balancing the page and its siblings, cells may be +** inserted into or removed from the parent page (pParent). Doing so +** may cause the parent page to become overfull or underfull. If this +** happens, it is the responsibility of the caller to invoke the correct +** balancing routine to fix this problem (see the balance() routine). +** +** If this routine fails for any reason, it might leave the database +** in a corrupted state. So if this routine fails, the database should +** be rolled back. +** +** The third argument to this function, aOvflSpace, is a pointer to a +** buffer big enough to hold one page. If while inserting cells into the parent +** page (pParent) the parent page becomes overfull, this buffer is +** used to store the parent's overflow cells. Because this function inserts +** a maximum of four divider cells into the parent page, and the maximum +** size of a cell stored within an internal node is always less than 1/4 +** of the page-size, the aOvflSpace[] buffer is guaranteed to be large +** enough for all overflow cells. +** +** If aOvflSpace is set to a null pointer, this function returns +** SQLITE_NOMEM. */ -static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ - Pager * const pDestPager = sqlite3BtreePager(p->pDest); - const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); - int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); - const int nCopy = MIN(nSrcPgsz, nDestPgsz); - const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; +static int balance_nonroot( + MemPage *pParent, /* Parent page of siblings being balanced */ + int iParentIdx, /* Index of "the page" in pParent */ + u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ + int isRoot /* True if pParent is a root-page */ +){ + BtShared *pBt; /* The whole database */ + int nCell = 0; /* Number of cells in apCell[] */ + int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ + int nNew = 0; /* Number of pages in apNew[] */ + int nOld; /* Number of pages in apOld[] */ + int i, j, k; /* Loop counters */ + int nxDiv; /* Next divider slot in pParent->aCell[] */ + int rc = SQLITE_OK; /* The return code */ + u16 leafCorrection; /* 4 if pPage is a leaf. 0 if not */ + int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ + int usableSpace; /* Bytes in pPage beyond the header */ + int pageFlags; /* Value of pPage->aData[0] */ + int subtotal; /* Subtotal of bytes in cells on one page */ + int iSpace1 = 0; /* First unused byte of aSpace1[] */ + int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ + int szScratch; /* Size of scratch memory requested */ + MemPage *apOld[NB]; /* pPage and up to two siblings */ + MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ + MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ + u8 *pRight; /* Location in parent of right-sibling pointer */ + u8 *apDiv[NB-1]; /* Divider cells in pParent */ + int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int szNew[NB+2]; /* Combined size of cells place on i-th page */ + u8 **apCell = 0; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *aSpace1; /* Space for copies of dividers cells */ + Pgno pgno; /* Temp var to store a page number in */ - int rc = SQLITE_OK; - i64 iOff; + pBt = pParent->pBt; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - assert( p->bDestLocked ); - assert( !isFatalError(p->rc) ); - assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); - assert( zSrcData ); +#if 0 + TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); +#endif - /* Catch the case where the destination is an in-memory database and the - ** page sizes of the source and destination differ. + /* At this point pParent may have at most one overflow cell. And if + ** this overflow cell is present, it must be the cell with + ** index iParentIdx. This scenario comes about when this function + ** is called (indirectly) from sqlite3BtreeDelete(). */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ - rc = SQLITE_READONLY; - } + assert( pParent->nOverflow==0 || pParent->nOverflow==1 ); + assert( pParent->nOverflow==0 || pParent->aOvfl[0].idx==iParentIdx ); -#ifdef SQLITE_HAS_CODEC - /* Backup is not possible if the page size of the destination is changing - ** a a codec is in use. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ - rc = SQLITE_READONLY; + if( !aOvflSpace ){ + return SQLITE_NOMEM; } -#endif - /* This loop runs once for each destination page spanned by the source - ** page. For each iteration, variable iOff is set to the byte offset - ** of the destination page. + /* Find the sibling pages to balance. Also locate the cells in pParent + ** that divide the siblings. An attempt is made to find NN siblings on + ** either side of pPage. More siblings are taken from one side, however, + ** if there are fewer than NN siblings on the other side. If pParent + ** has NB or fewer children then all children of pParent are taken. + ** + ** This loop also drops the divider cells from the parent page. This + ** way, the remainder of the function does not have to deal with any + ** overflow cells in the parent page, since if any existed they will + ** have already been removed. */ - for(iOff=iEnd-(i64)nSrcPgsz; rc==SQLITE_OK && iOffpDest->pBt) ) continue; - if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) - && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) - ){ - const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; - u8 *zDestData = sqlite3PagerGetData(pDestPg); - u8 *zOut = &zDestData[iOff%nDestPgsz]; - - /* Copy the data from the source page into the destination page. - ** Then clear the Btree layer MemPage.isInit flag. Both this module - ** and the pager code use this trick (clearing the first byte - ** of the page 'extra' space to invalidate the Btree layers - ** cached parse of the page). MemPage.isInit is marked - ** "MUST BE FIRST" for this purpose. - */ - memcpy(zOut, zIn, nCopy); - ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + i = pParent->nOverflow + pParent->nCell; + if( i<2 ){ + nxDiv = 0; + nOld = i+1; + }else{ + nOld = 3; + if( iParentIdx==0 ){ + nxDiv = 0; + }else if( iParentIdx==i ){ + nxDiv = i-2; + }else{ + nxDiv = iParentIdx-1; } - sqlite3PagerUnref(pDestPg); + i = 2; + } + if( (i+nxDiv-pParent->nOverflow)==pParent->nCell ){ + pRight = &pParent->aData[pParent->hdrOffset+8]; + }else{ + pRight = findCell(pParent, i+nxDiv-pParent->nOverflow); } + pgno = get4byte(pRight); + while( 1 ){ + rc = getAndInitPage(pBt, pgno, &apOld[i]); + if( rc ){ + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + } + nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; + if( (i--)==0 ) break; - return rc; -} + if( i+nxDiv==pParent->aOvfl[0].idx && pParent->nOverflow ){ + apDiv[i] = pParent->aOvfl[0].pCell; + pgno = get4byte(apDiv[i]); + szNew[i] = cellSizePtr(pParent, apDiv[i]); + pParent->nOverflow = 0; + }else{ + apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); + pgno = get4byte(apDiv[i]); + szNew[i] = cellSizePtr(pParent, apDiv[i]); -/* -** If pFile is currently larger than iSize bytes, then truncate it to -** exactly iSize bytes. If pFile is not larger than iSize bytes, then -** this function is a no-op. -** -** Return SQLITE_OK if everything is successful, or an SQLite error -** code if an error occurs. -*/ -static int backupTruncateFile(sqlite3_file *pFile, i64 iSize){ - i64 iCurrent; - int rc = sqlite3OsFileSize(pFile, &iCurrent); - if( rc==SQLITE_OK && iCurrent>iSize ){ - rc = sqlite3OsTruncate(pFile, iSize); + /* Drop the cell from the parent page. apDiv[i] still points to + ** the cell within the parent, even though it has been dropped. + ** This is safe because dropping a cell only overwrites the first + ** four bytes of it, and this function does not need the first + ** four bytes of the divider cell. So the pointer is safe to use + ** later on. + ** + ** Unless SQLite is compiled in secure-delete mode. In this case, + ** the dropCell() routine will overwrite the entire cell with zeroes. + ** In this case, temporarily copy the cell into the aOvflSpace[] + ** buffer. It will be copied out again as soon as the aSpace[] buffer + ** is allocated. */ + if( pBt->secureDelete ){ + int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); + if( (iOff+szNew[i])>(int)pBt->usableSize ){ + rc = SQLITE_CORRUPT_BKPT; + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + }else{ + memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); + apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; + } + } + dropCell(pParent, i+nxDiv-pParent->nOverflow, szNew[i], &rc); + } } - return rc; -} - -/* -** Register this backup object with the associated source pager for -** callbacks when pages are changed or the cache invalidated. -*/ -static void attachBackupObject(sqlite3_backup *p){ - sqlite3_backup **pp; - assert( sqlite3BtreeHoldsMutex(p->pSrc) ); - pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); - p->pNext = *pp; - *pp = p; - p->isAttached = 1; -} -/* -** Copy nPage pages from the source b-tree to the destination. -*/ -SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ - int rc; - int destMode; /* Destination journal mode */ - int pgszSrc = 0; /* Source page size */ - int pgszDest = 0; /* Destination page size */ + /* Make nMaxCells a multiple of 4 in order to preserve 8-byte + ** alignment */ + nMaxCells = (nMaxCells + 3)&~3; - sqlite3_mutex_enter(p->pSrcDb->mutex); - sqlite3BtreeEnter(p->pSrc); - if( p->pDestDb ){ - sqlite3_mutex_enter(p->pDestDb->mutex); + /* + ** Allocate space for memory structures + */ + k = pBt->pageSize + ROUND8(sizeof(MemPage)); + szScratch = + nMaxCells*sizeof(u8*) /* apCell */ + + nMaxCells*sizeof(u16) /* szCell */ + + pBt->pageSize /* aSpace1 */ + + k*nOld; /* Page copies (apCopy) */ + apCell = sqlite3ScratchMalloc( szScratch ); + if( apCell==0 ){ + rc = SQLITE_NOMEM; + goto balance_cleanup; } + szCell = (u16*)&apCell[nMaxCells]; + aSpace1 = (u8*)&szCell[nMaxCells]; + assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); - rc = p->rc; - if( !isFatalError(rc) ){ - Pager * const pSrcPager = sqlite3BtreePager(p->pSrc); /* Source pager */ - Pager * const pDestPager = sqlite3BtreePager(p->pDest); /* Dest pager */ - int ii; /* Iterator variable */ - int nSrcPage = -1; /* Size of source db in pages */ - int bCloseTrans = 0; /* True if src db requires unlocking */ + /* + ** Load pointers to all cells on sibling pages and the divider cells + ** into the local apCell[] array. Make copies of the divider cells + ** into space obtained from aSpace1[] and remove the the divider Cells + ** from pParent. + ** + ** If the siblings are on leaf pages, then the child pointers of the + ** divider cells are stripped from the cells before they are copied + ** into aSpace1[]. In this way, all cells in apCell[] are without + ** child pointers. If siblings are not leaves, then all cell in + ** apCell[] include child pointers. Either way, all cells in apCell[] + ** are alike. + ** + ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. + ** leafData: 1 if pPage holds key+data and pParent holds only keys. + */ + leafCorrection = apOld[0]->leaf*4; + leafData = apOld[0]->hasData; + for(i=0; ipageSize + k*i]; + memcpy(pOld, apOld[i], sizeof(MemPage)); + pOld->aData = (void*)&pOld[1]; + memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); - /* If the source pager is currently in a write-transaction, return - ** SQLITE_BUSY immediately. - */ - if( p->pDestDb && p->pSrc->pBt->inTransaction==TRANS_WRITE ){ - rc = SQLITE_BUSY; + limit = pOld->nCell+pOld->nOverflow; + if( pOld->nOverflow>0 ){ + for(j=0; jaData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; jmaxLocal+23 ); + assert( iSpace1 <= (int)pBt->pageSize ); + memcpy(pTemp, apDiv[i], sz); + apCell[nCell] = pTemp+leafCorrection; + assert( leafCorrection==0 || leafCorrection==4 ); + szCell[nCell] = szCell[nCell] - leafCorrection; + if( !pOld->leaf ){ + assert( leafCorrection==0 ); + assert( pOld->hdrOffset==0 ); + /* The right pointer of the child page pOld becomes the left + ** pointer of the divider cell */ + memcpy(apCell[nCell], &pOld->aData[8], 4); + }else{ + assert( leafCorrection==4 ); + if( szCell[nCell]<4 ){ + /* Do not allow any cells smaller than 4 bytes. */ + szCell[nCell] = 4; + } + } + nCell++; } + } - /* Lock the destination database, if it is not locked already. */ - if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) - ){ - p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + /* + ** Figure out the number of pages needed to hold all nCell cells. + ** Store this number in "k". Also compute szNew[] which is the total + ** size of all cells on the i-th page and cntNew[] which is the index + ** in apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal nCell. + ** + ** Values computed by this block: + ** + ** k: The total number of sibling pages + ** szNew[i]: Spaced used on the i-th sibling page. + ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to + ** the right of the i-th sibling page. + ** usableSpace: Number of bytes of space available on each sibling. + ** + */ + usableSpace = pBt->usableSize - 12 + leafCorrection; + for(subtotal=k=i=0; i usableSpace ){ + szNew[k] = subtotal - szCell[i]; + cntNew[k] = i; + if( leafData ){ i--; } + subtotal = 0; + k++; + if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } } + } + szNew[k] = subtotal; + cntNew[k] = nCell; + k++; - /* If there is no open read-transaction on the source database, open - ** one now. If a transaction is opened here, then it will be closed - ** before this function exits. - */ - if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ - rc = sqlite3BtreeBeginTrans(p->pSrc, 0); - bCloseTrans = 1; - } + /* + ** The packing computed by the previous block is biased toward the siblings + ** on the left side. The left siblings are always nearly full, while the + ** right-most sibling might be nearly empty. This block of code attempts + ** to adjust the packing of siblings to get a better balance. + ** + ** This adjustment is more than an optimization. The packing above might + ** be so out of balance as to be illegal. For example, the right-most + ** sibling might be completely empty. This adjustment is not optional. + */ + for(i=k-1; i>0; i--){ + int szRight = szNew[i]; /* Size of sibling on the right */ + int szLeft = szNew[i-1]; /* Size of sibling on the left */ + int r; /* Index of right-most cell in left sibling */ + int d; /* Index of first cell to the left of right sibling */ - /* Do not allow backup if the destination database is in WAL mode - ** and the page sizes are different between source and destination */ - pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); - pgszDest = sqlite3BtreeGetPageSize(p->pDest); - destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); - if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ - rc = SQLITE_READONLY; - } - - /* Now that there is a read-lock on the source database, query the - ** source pager for the number of pages in the database. - */ - nSrcPage = (int)sqlite3BtreeLastPage(p->pSrc); - assert( nSrcPage>=0 ); - for(ii=0; (nPage<0 || iiiNext<=(Pgno)nSrcPage && !rc; ii++){ - const Pgno iSrcPg = p->iNext; /* Source page number */ - if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ - DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); - if( rc==SQLITE_OK ){ - rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg)); - sqlite3PagerUnref(pSrcPg); - } - } - p->iNext++; - } - if( rc==SQLITE_OK ){ - p->nPagecount = nSrcPage; - p->nRemaining = nSrcPage+1-p->iNext; - if( p->iNext>(Pgno)nSrcPage ){ - rc = SQLITE_DONE; - }else if( !p->isAttached ){ - attachBackupObject(p); - } + r = cntNew[i-1] - 1; + d = r + 1 - leafData; + assert( dpDest,1,p->iDestSchema+1))==SQLITE_OK - ){ - int nDestTruncate; - - if( p->pDestDb ){ - sqlite3ResetInternalSchema(p->pDestDb, 0); - } + szNew[i] = szRight; + szNew[i-1] = szLeft; + } - /* Set nDestTruncate to the final number of pages in the destination - ** database. The complication here is that the destination page - ** size may be different to the source page size. - ** - ** If the source page size is smaller than the destination page size, - ** round up. In this case the call to sqlite3OsTruncate() below will - ** fix the size of the file. However it is important to call - ** sqlite3PagerTruncateImage() here so that any pages in the - ** destination file that lie beyond the nDestTruncate page mark are - ** journalled by PagerCommitPhaseOne() before they are destroyed - ** by the file truncation. - */ - assert( pgszSrc==sqlite3BtreeGetPageSize(p->pSrc) ); - assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); - if( pgszSrcpDest->pBt) ){ - nDestTruncate--; - } - }else{ - nDestTruncate = nSrcPage * (pgszSrc/pgszDest); - } - sqlite3PagerTruncateImage(pDestPager, nDestTruncate); + /* Either we found one or more cells (cntnew[0])>0) or pPage is + ** a virtual root page. A virtual root page is when the real root + ** page is page 1 and we are the only child of that page. + */ + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); - if( pgszSrcpgno, + nOld>=2 ? apOld[1]->pgno : 0, + nOld>=3 ? apOld[2]->pgno : 0 + )); - assert( pFile ); - assert( (i64)nDestTruncate*(i64)pgszDest >= iSize || ( - nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) - && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest - )); - if( SQLITE_OK==(rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1)) - && SQLITE_OK==(rc = backupTruncateFile(pFile, iSize)) - && SQLITE_OK==(rc = sqlite3PagerSync(pDestPager)) - ){ - i64 iOff; - i64 iEnd = MIN(PENDING_BYTE + pgszDest, iSize); - for( - iOff=PENDING_BYTE+pgszSrc; - rc==SQLITE_OK && iOffpgno<=1 ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + pageFlags = apOld[0]->aData[0]; + for(i=0; ipDbPage); + nNew++; + if( rc ) goto balance_cleanup; + }else{ + assert( i>0 ); + rc = allocateBtreePage(pBt, &pNew, &pgno, pgno, 0); + if( rc ) goto balance_cleanup; + apNew[i] = pNew; + nNew++; + + /* Set the pointer-map entry for the new sibling page. */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; } - }else{ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); - } - - /* Finish committing the transaction to the destination database. */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3BtreeCommitPhaseTwo(p->pDest)) - ){ - rc = SQLITE_DONE; } } - - /* If bCloseTrans is true, then this function opened a read transaction - ** on the source database. Close the read transaction here. There is - ** no need to check the return values of the btree methods here, as - ** "committing" a read-only transaction cannot fail. - */ - if( bCloseTrans ){ - TESTONLY( int rc2 ); - TESTONLY( rc2 = ) sqlite3BtreeCommitPhaseOne(p->pSrc, 0); - TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc); - assert( rc2==SQLITE_OK ); - } - - if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; - } - p->rc = rc; } - if( p->pDestDb ){ - sqlite3_mutex_leave(p->pDestDb->mutex); - } - sqlite3BtreeLeave(p->pSrc); - sqlite3_mutex_leave(p->pSrcDb->mutex); - return rc; -} - -/* -** Release all resources associated with an sqlite3_backup* handle. -*/ -SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ - sqlite3_backup **pp; /* Ptr to head of pagers backup list */ - sqlite3_mutex *mutex; /* Mutex to protect source database */ - int rc; /* Value to return */ - /* Enter the mutexes */ - if( p==0 ) return SQLITE_OK; - sqlite3_mutex_enter(p->pSrcDb->mutex); - sqlite3BtreeEnter(p->pSrc); - mutex = p->pSrcDb->mutex; - if( p->pDestDb ){ - sqlite3_mutex_enter(p->pDestDb->mutex); + /* Free any old pages that were not reused as new pages. + */ + while( ipDestDb ){ - p->pSrc->nBackup--; - } - if( p->isAttached ){ - pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); - while( *pp!=p ){ - pp = &(*pp)->pNext; + /* + ** Put the new pages in accending order. This helps to + ** keep entries in the disk file in order so that a scan + ** of the table is a linear scan through the file. That + ** in turn helps the operating system to deliver pages + ** from the disk more rapidly. + ** + ** An O(n^2) insertion sort algorithm is used, but since + ** n is never more than NB (a small constant), that should + ** not be a problem. + ** + ** When NB==3, this one optimization makes the database + ** about 25% faster for large insertions and deletions. + */ + for(i=0; ipgno; + int minI = i; + for(j=i+1; jpgno<(unsigned)minV ){ + minI = j; + minV = apNew[j]->pgno; + } + } + if( minI>i ){ + MemPage *pT; + pT = apNew[i]; + apNew[i] = apNew[minI]; + apNew[minI] = pT; } - *pp = p->pNext; } + TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", + apNew[0]->pgno, szNew[0], + nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); - /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest); - - /* Set the error code of the destination database handle. */ - rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; - sqlite3Error(p->pDestDb, rc, 0); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + put4byte(pRight, apNew[nNew-1]->pgno); - /* Exit the mutexes and free the backup context structure. */ - if( p->pDestDb ){ - sqlite3_mutex_leave(p->pDestDb->mutex); - } - sqlite3BtreeLeave(p->pSrc); - if( p->pDestDb ){ - /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a - ** call to sqlite3_backup_init() and is destroyed by a call to - ** sqlite3_backup_finish(). */ - sqlite3_free(p); - } - sqlite3_mutex_leave(mutex); - return rc; -} + /* + ** Evenly distribute the data in apCell[] across the new pages. + ** Insert divider cells into pParent as necessary. + */ + j = 0; + for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); + assert( pNew->nOverflow==0 ); -/* -** Return the number of pages still to be backed up as of the most recent -** call to sqlite3_backup_step(). -*/ -SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ - return p->nRemaining; -} + j = cntNew[i]; -/* -** Return the total number of pages in the source database as of the most -** recent call to sqlite3_backup_step(). -*/ -SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ - return p->nPagecount; -} + /* If the sibling page assembled above was not the right-most sibling, + ** insert a divider cell into the parent page. + */ + assert( ipNext){ - assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); - if( !isFatalError(p->rc) && iPageiNext ){ - /* The backup process p has already copied page iPage. But now it - ** has been modified by a transaction on the source pager. Copy - ** the new data into the backup. - */ - int rc = backupOnePage(p, iPage, aData); - assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); - if( rc!=SQLITE_OK ){ - p->rc = rc; + assert( jleaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + + j++; + nxDiv++; } } -} + assert( j==nCell ); + assert( nOld>0 ); + assert( nNew>0 ); + if( (pageFlags & PTF_LEAF)==0 ){ + u8 *zChild = &apCopy[nOld-1]->aData[8]; + memcpy(&apNew[nNew-1]->aData[8], zChild, 4); + } -/* -** Restart the backup process. This is called when the pager layer -** detects that the database has been modified by an external database -** connection. In this case there is no way of knowing which of the -** pages that have been copied into the destination database are still -** valid and which are not, so the entire process needs to be restarted. -** -** It is assumed that the mutex associated with the BtShared object -** corresponding to the source database is held when this function is -** called. -*/ -SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *pBackup){ - sqlite3_backup *p; /* Iterator variable */ - for(p=pBackup; p; p=p->pNext){ - assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); - p->iNext = 1; - } -} + if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ + /* The root page of the b-tree now contains no cells. The only sibling + ** page is the right-child of the parent. Copy the contents of the + ** child page into the parent, decreasing the overall height of the + ** b-tree structure by one. This is described as the "balance-shallower" + ** sub-algorithm in some documentation. + ** + ** If this is an auto-vacuum database, the call to copyNodeContent() + ** sets all pointer-map entries corresponding to database image pages + ** for which the pointer is stored within the content being copied. + ** + ** The second assert below verifies that the child page is defragmented + ** (it must be, as it was just reconstructed using assemblePage()). This + ** is important if the parent page happens to be page 1 of the database + ** image. */ + assert( nNew==1 ); + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + }else if( ISAUTOVACUUM ){ + /* Fix the pointer-map entries for all the cells that were shifted around. + ** There are several different types of pointer-map entries that need to + ** be dealt with by this routine. Some of these have been set already, but + ** many have not. The following is a summary: + ** + ** 1) The entries associated with new sibling pages that were not + ** siblings when this function was called. These have already + ** been set. We don't need to worry about old siblings that were + ** moved to the free-list - the freePage() code has taken care + ** of those. + ** + ** 2) The pointer-map entries associated with the first overflow + ** page in any overflow chains used by new divider cells. These + ** have also already been taken care of by the insertCell() code. + ** + ** 3) If the sibling pages are not leaves, then the child pages of + ** cells stored on the sibling pages may need to be updated. + ** + ** 4) If the sibling pages are not internal intkey nodes, then any + ** overflow pages used by these cells may need to be updated + ** (internal intkey nodes never contain pointers to overflow pages). + ** + ** 5) If the sibling pages are not leaves, then the pointer-map + ** entries for the right-child pages of each sibling may need + ** to be updated. + ** + ** Cases 1 and 2 are dealt with above by other code. The next + ** block deals with cases 3 and 4 and the one after that, case 5. Since + ** setting a pointer map entry is a relatively expensive operation, this + ** code only sets pointer map entries for child or overflow pages that have + ** actually moved between pages. */ + MemPage *pNew = apNew[0]; + MemPage *pOld = apCopy[0]; + int nOverflow = pOld->nOverflow; + int iNextOld = pOld->nCell + nOverflow; + int iOverflow = (nOverflow ? pOld->aOvfl[0].idx : -1); + j = 0; /* Current 'old' sibling page */ + k = 0; /* Current 'new' sibling page */ + for(i=0; inCell + pOld->nOverflow; + if( pOld->nOverflow ){ + nOverflow = pOld->nOverflow; + iOverflow = i + !leafData + pOld->aOvfl[0].idx; + } + isDivider = !leafData; + } -#ifndef SQLITE_OMIT_VACUUM -/* -** Copy the complete content of pBtFrom into pBtTo. A transaction -** must be active for both files. -** -** The size of file pTo may be reduced by this operation. If anything -** goes wrong, the transaction on pTo is rolled back. If successful, the -** transaction is committed before returning. -*/ -SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ - int rc; - sqlite3_backup b; - sqlite3BtreeEnter(pTo); - sqlite3BtreeEnter(pFrom); + assert(nOverflow>0 || iOverflowaOvfl[0].idx==pOld->aOvfl[1].idx-1); + assert(nOverflow<3 || pOld->aOvfl[1].idx==pOld->aOvfl[2].idx-1); + if( i==iOverflow ){ + isDivider = 1; + if( (--nOverflow)>0 ){ + iOverflow++; + } + } - /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set - ** to 0. This is used by the implementations of sqlite3_backup_step() - ** and sqlite3_backup_finish() to detect that they are being called - ** from this function, not directly by the user. - */ - memset(&b, 0, sizeof(b)); - b.pSrcDb = pFrom->db; - b.pSrc = pFrom; - b.pDest = pTo; - b.iNext = 1; + if( i==cntNew[k] ){ + /* Cell i is the cell immediately following the last cell on new + ** sibling page k. If the siblings are not leaf pages of an + ** intkey b-tree, then cell i is a divider cell. */ + pNew = apNew[++k]; + if( !leafData ) continue; + } + assert( jrc) should be set to either SQLITE_DONE - ** or an error code. + /* If the cell was originally divider cell (and is not now) or + ** an overflow cell, or if the cell was located on a different sibling + ** page before the balancing, then the pointer map entries associated + ** with any child or overflow pages need to be updated. */ + if( isDivider || pOld->pgno!=pNew->pgno ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, apCell[i], &rc); + } + } + } + + if( !leafCorrection ){ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); + } + } + +#if 0 + /* The ptrmapCheckPages() contains assert() statements that verify that + ** all pointer map pages are set correctly. This is helpful while + ** debugging. This is usually disabled because a corrupt database may + ** cause an assert() statement to fail. */ + ptrmapCheckPages(apNew, nNew); + ptrmapCheckPages(&pParent, 1); +#endif + } + + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); + + /* + ** Cleanup before returning. */ - sqlite3_backup_step(&b, 0x7FFFFFFF); - assert( b.rc!=SQLITE_OK ); - rc = sqlite3_backup_finish(&b); - if( rc==SQLITE_OK ){ - pTo->pBt->pageSizeFixed = 0; +balance_cleanup: + sqlite3ScratchFree(apCell); + for(i=0; ipBt; /* The BTree */ -/* -** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*) -** P if required. -*/ -#define expandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) + assert( pRoot->nOverflow>0 ); + assert( sqlite3_mutex_held(pBt->mutex) ); -/* -** If pMem is an object with a valid string representation, this routine -** ensures the internal encoding for the string representation is -** 'desiredEnc', one of SQLITE_UTF8, SQLITE_UTF16LE or SQLITE_UTF16BE. -** -** If pMem is not a string object, or the encoding of the string -** representation is already stored using the requested encoding, then this -** routine is a no-op. -** -** SQLITE_OK is returned if the conversion is successful (or not required). -** SQLITE_NOMEM may be returned if a malloc() fails during conversion -** between formats. -*/ -SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ - int rc; - assert( (pMem->flags&MEM_RowSet)==0 ); - assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE - || desiredEnc==SQLITE_UTF16BE ); - if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ - return SQLITE_OK; + /* Make pRoot, the root page of the b-tree, writable. Allocate a new + ** page that will become the new right-child of pPage. Copy the contents + ** of the node stored on pRoot into the new child page. + */ + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc==SQLITE_OK ){ + rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); + copyNodeContent(pRoot, pChild, &rc); + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); + } } - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); -#ifdef SQLITE_OMIT_UTF16 - return SQLITE_ERROR; -#else + if( rc ){ + *ppChild = 0; + releasePage(pChild); + return rc; + } + assert( sqlite3PagerIswriteable(pChild->pDbPage) ); + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + assert( pChild->nCell==pRoot->nCell ); - /* MemTranslate() may return SQLITE_OK or SQLITE_NOMEM. If NOMEM is returned, - ** then the encoding of the value may not have changed. - */ - rc = sqlite3VdbeMemTranslate(pMem, (u8)desiredEnc); - assert(rc==SQLITE_OK || rc==SQLITE_NOMEM); - assert(rc==SQLITE_OK || pMem->enc!=desiredEnc); - assert(rc==SQLITE_NOMEM || pMem->enc==desiredEnc); - return rc; -#endif + TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + + /* Copy the overflow cells from pRoot to pChild */ + memcpy(pChild->aOvfl, pRoot->aOvfl, pRoot->nOverflow*sizeof(pRoot->aOvfl[0])); + pChild->nOverflow = pRoot->nOverflow; + + /* Zero the contents of pRoot. Then install pChild as the right-child. */ + zeroPage(pRoot, pChild->aData[0] & ~PTF_LEAF); + put4byte(&pRoot->aData[pRoot->hdrOffset+8], pgnoChild); + + *ppChild = pChild; + return SQLITE_OK; } /* -** Make sure pMem->z points to a writable allocation of at least -** n bytes. -** -** If the memory cell currently contains string or blob data -** and the third argument passed to this function is true, the -** current content of the cell is preserved. Otherwise, it may -** be discarded. +** The page that pCur currently points to has just been modified in +** some way. This function figures out if this modification means the +** tree needs to be balanced, and if so calls the appropriate balancing +** routine. Balancing routines are: ** -** This function sets the MEM_Dyn flag and clears any xDel callback. -** It also clears MEM_Ephem and MEM_Static. If the preserve flag is -** not set, Mem.n is zeroed. +** balance_quick() +** balance_deeper() +** balance_nonroot() */ -SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve){ - assert( 1 >= - ((pMem->zMalloc && pMem->zMalloc==pMem->z) ? 1 : 0) + - (((pMem->flags&MEM_Dyn)&&pMem->xDel) ? 1 : 0) + - ((pMem->flags&MEM_Ephem) ? 1 : 0) + - ((pMem->flags&MEM_Static) ? 1 : 0) - ); - assert( (pMem->flags&MEM_RowSet)==0 ); +static int balance(BtCursor *pCur){ + int rc = SQLITE_OK; + const int nMin = pCur->pBt->usableSize * 2 / 3; + u8 aBalanceQuickSpace[13]; + u8 *pFree = 0; - if( n<32 ) n = 32; - if( sqlite3DbMallocSize(pMem->db, pMem->zMalloc)z==pMem->zMalloc ){ - pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); - preserve = 0; - }else{ - sqlite3DbFree(pMem->db, pMem->zMalloc); - pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); - } - } + TESTONLY( int balance_quick_called = 0 ); + TESTONLY( int balance_deeper_called = 0 ); - if( pMem->z && preserve && pMem->zMalloc && pMem->z!=pMem->zMalloc ){ - memcpy(pMem->zMalloc, pMem->z, pMem->n); - } - if( pMem->flags&MEM_Dyn && pMem->xDel ){ - pMem->xDel((void *)(pMem->z)); - } + do { + int iPage = pCur->iPage; + MemPage *pPage = pCur->apPage[iPage]; - pMem->z = pMem->zMalloc; - if( pMem->z==0 ){ - pMem->flags = MEM_Null; - }else{ - pMem->flags &= ~(MEM_Ephem|MEM_Static); - } - pMem->xDel = 0; - return (pMem->z ? SQLITE_OK : SQLITE_NOMEM); -} + if( iPage==0 ){ + if( pPage->nOverflow ){ + /* The root page of the b-tree is overfull. In this case call the + ** balance_deeper() function to create a new child for the root-page + ** and copy the current contents of the root-page to it. The + ** next iteration of the do-loop will balance the child page. + */ + assert( (balance_deeper_called++)==0 ); + rc = balance_deeper(pPage, &pCur->apPage[1]); + if( rc==SQLITE_OK ){ + pCur->iPage = 1; + pCur->aiIdx[0] = 0; + pCur->aiIdx[1] = 0; + assert( pCur->apPage[1]->nOverflow ); + } + }else{ + break; + } + }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + break; + }else{ + MemPage * const pParent = pCur->apPage[iPage-1]; + int const iIdx = pCur->aiIdx[iPage-1]; -/* -** Make the given Mem object MEM_Dyn. In other words, make it so -** that any TEXT or BLOB content is stored in memory obtained from -** malloc(). In this way, we know that the memory is safe to be -** overwritten or altered. -** -** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ - int f; - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags&MEM_RowSet)==0 ); - expandBlob(pMem); - f = pMem->flags; - if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ - if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ - return SQLITE_NOMEM; - } - pMem->z[pMem->n] = 0; - pMem->z[pMem->n+1] = 0; - pMem->flags |= MEM_Term; -#ifdef SQLITE_DEBUG - pMem->pScopyFrom = 0; + rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK ){ +#ifndef SQLITE_OMIT_QUICKBALANCE + if( pPage->hasData + && pPage->nOverflow==1 + && pPage->aOvfl[0].idx==pPage->nCell + && pParent->pgno!=1 + && pParent->nCell==iIdx + ){ + /* Call balance_quick() to create a new sibling of pPage on which + ** to store the overflow cell. balance_quick() inserts a new cell + ** into pParent, which may cause pParent overflow. If this + ** happens, the next interation of the do-loop will balance pParent + ** use either balance_nonroot() or balance_deeper(). Until this + ** happens, the overflow cell is stored in the aBalanceQuickSpace[] + ** buffer. + ** + ** The purpose of the following assert() is to check that only a + ** single call to balance_quick() is made for each call to this + ** function. If this were not verified, a subtle bug involving reuse + ** of the aBalanceQuickSpace[] might sneak in. + */ + assert( (balance_quick_called++)==0 ); + rc = balance_quick(pParent, pPage, aBalanceQuickSpace); + }else #endif - } + { + /* In this case, call balance_nonroot() to redistribute cells + ** between pPage and up to 2 of its sibling pages. This involves + ** modifying the contents of pParent, which may cause pParent to + ** become overfull or underfull. The next iteration of the do-loop + ** will balance the parent page to correct this. + ** + ** If the parent page becomes overfull, the overflow cell or cells + ** are stored in the pSpace buffer allocated immediately below. + ** A subsequent iteration of the do-loop will deal with this by + ** calling balance_nonroot() (balance_deeper() may be called first, + ** but it doesn't deal with overflow cells - just moves them to a + ** different page). Once this subsequent call to balance_nonroot() + ** has completed, it is safe to release the pSpace buffer used by + ** the previous call, as the overflow cell data will have been + ** copied either into the body of a database page or into the new + ** pSpace buffer passed to the latter call to balance_nonroot(). + */ + u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1); + if( pFree ){ + /* If pFree is not NULL, it points to the pSpace buffer used + ** by a previous call to balance_nonroot(). Its contents are + ** now stored either on real database pages or within the + ** new pSpace buffer, so it may be safely freed here. */ + sqlite3PageFree(pFree); + } - return SQLITE_OK; -} + /* The pSpace buffer will be freed after the next call to + ** balance_nonroot(), or just before this function returns, whichever + ** comes first. */ + pFree = pSpace; + } + } -/* -** If the given Mem* has a zero-filled tail, turn it into an ordinary -** blob stored in dynamically allocated space. -*/ -#ifndef SQLITE_OMIT_INCRBLOB -SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ - if( pMem->flags & MEM_Zero ){ - int nByte; - assert( pMem->flags&MEM_Blob ); - assert( (pMem->flags&MEM_RowSet)==0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + pPage->nOverflow = 0; - /* Set nByte to the number of bytes required to store the expanded blob. */ - nByte = pMem->n + pMem->u.nZero; - if( nByte<=0 ){ - nByte = 1; - } - if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ - return SQLITE_NOMEM; + /* The next iteration of the do-loop balances the parent page. */ + releasePage(pPage); + pCur->iPage--; } + }while( rc==SQLITE_OK ); - memset(&pMem->z[pMem->n], 0, pMem->u.nZero); - pMem->n += pMem->u.nZero; - pMem->flags &= ~(MEM_Zero|MEM_Term); + if( pFree ){ + sqlite3PageFree(pFree); } - return SQLITE_OK; + return rc; } -#endif - -/* -** Make sure the given Mem is \u0000 terminated. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - if( (pMem->flags & MEM_Term)!=0 || (pMem->flags & MEM_Str)==0 ){ - return SQLITE_OK; /* Nothing to do */ - } - if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ - return SQLITE_NOMEM; - } - pMem->z[pMem->n] = 0; - pMem->z[pMem->n+1] = 0; - pMem->flags |= MEM_Term; - return SQLITE_OK; -} /* -** Add MEM_Str to the set of representations for the given Mem. Numbers -** are converted using sqlite3_snprintf(). Converting a BLOB to a string -** is a no-op. +** Insert a new record into the BTree. The key is given by (pKey,nKey) +** and the data is given by (pData,nData). The cursor is used only to +** define what table the record should be inserted into. The cursor +** is left pointing at a random location. ** -** Existing representations MEM_Int and MEM_Real are *not* invalidated. +** For an INTKEY table, only the nKey value of the key is used. pKey is +** ignored. For a ZERODATA table, the pData and nData are both ignored. ** -** A MEM_Null value will never be passed to this function. This function is -** used for converting values to text for returning to the user (i.e. via -** sqlite3_value_text()), or for ensuring that values to be used as btree -** keys are strings. In the former case a NULL pointer is returned the -** user and the later is an internal programming error. +** If the seekResult parameter is non-zero, then a successful call to +** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already +** been performed. seekResult is the search result returned (a negative +** number if pCur points at an entry that is smaller than (pKey, nKey), or +** a positive value if pCur points at an etry that is larger than +** (pKey, nKey)). +** +** If the seekResult parameter is non-zero, then the caller guarantees that +** cursor pCur is pointing at the existing copy of a row that is to be +** overwritten. If the seekResult parameter is 0, then cursor pCur may +** point to any entry or to no entry at all and so this function has to seek +** the cursor before the new key can be inserted. */ -SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, int enc){ - int rc = SQLITE_OK; - int fg = pMem->flags; - const int nByte = 32; +SQLITE_PRIVATE int sqlite3BtreeInsert( + BtCursor *pCur, /* Insert data into the table of this cursor */ + const void *pKey, i64 nKey, /* The key of the new record */ + const void *pData, int nData, /* The data of the new record */ + int nZero, /* Number of extra 0 bytes to append to data */ + int appendBias, /* True if this is likely an append */ + int seekResult /* Result of prior MovetoUnpacked() call */ +){ + int rc; + int loc = seekResult; /* -1: before desired location +1: after */ + int szNew = 0; + int idx; + MemPage *pPage; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + unsigned char *oldCell; + unsigned char *newCell = 0; - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); - assert( (pMem->flags&MEM_RowSet)==0 ); - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + assert( cursorHoldsMutex(pCur) ); + assert( pCur->wrFlag && pBt->inTransaction==TRANS_WRITE && !pBt->readOnly ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); - if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ - return SQLITE_NOMEM; + /* Assert that the caller has been consistent. If this cursor was opened + ** expecting an index b-tree, then the caller should be inserting blob + ** keys with no associated data. If the cursor was opened expecting an + ** intkey table, the caller should be inserting integer keys with a + ** blob of associated data. */ + assert( (pKey==0)==(pCur->pKeyInfo==0) ); + + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced (assuming this is a replace + ** operation - if it is not, the following is a no-op). */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, nKey, 0); } - /* For a Real or Integer, use sqlite3_mprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. + /* Save the positions of any other cursors open on this table. + ** + ** In some cases, the call to btreeMoveto() below is a no-op. For + ** example, when inserting data into a table with auto-generated integer + ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the + ** integer key to use. It then calls this function to actually insert the + ** data into the intkey B-Tree. In this case btreeMoveto() recognizes + ** that the cursor is already where it needs to be and returns without + ** doing any work. To avoid thwarting these optimizations, it is important + ** not to clear the cursor here. */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->r); + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + if( !loc ){ + rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); + if( rc ) return rc; } - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - pMem->flags |= MEM_Str|MEM_Term; - sqlite3VdbeChangeEncoding(pMem, enc); - return rc; -} + assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); -/* -** Memory cell pMem contains the context of an aggregate function. -** This routine calls the finalize method for that function. The -** result of the aggregate is stored back into pMem. -** -** Return SQLITE_ERROR if the finalizer reports an error. SQLITE_OK -** otherwise. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ - int rc = SQLITE_OK; - if( ALWAYS(pFunc && pFunc->xFinalize) ){ - sqlite3_context ctx; - assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - memset(&ctx, 0, sizeof(ctx)); - ctx.s.flags = MEM_Null; - ctx.s.db = pMem->db; - ctx.pMem = pMem; - ctx.pFunc = pFunc; - pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ - assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); - sqlite3DbFree(pMem->db, pMem->zMalloc); - memcpy(pMem, &ctx.s, sizeof(ctx.s)); - rc = ctx.isError; - } - return rc; -} + pPage = pCur->apPage[pCur->iPage]; + assert( pPage->intKey || nKey>=0 ); + assert( pPage->leaf || !pPage->intKey ); -/* -** If the memory cell contains a string value that must be freed by -** invoking an external callback, free it now. Calling this function -** does not free any Mem.zMalloc buffer. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p){ - assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); - testcase( p->flags & MEM_Agg ); - testcase( p->flags & MEM_Dyn ); - testcase( p->flags & MEM_RowSet ); - testcase( p->flags & MEM_Frame ); - if( p->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame) ){ - if( p->flags&MEM_Agg ){ - sqlite3VdbeMemFinalize(p, p->u.pDef); - assert( (p->flags & MEM_Agg)==0 ); - sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn && p->xDel ){ - assert( (p->flags&MEM_RowSet)==0 ); - p->xDel((void *)p->z); - p->xDel = 0; - }else if( p->flags&MEM_RowSet ){ - sqlite3RowSetClear(p->u.pRowSet); - }else if( p->flags&MEM_Frame ){ - sqlite3VdbeMemSetNull(p); + TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + pCur->pgnoRoot, nKey, nData, pPage->pgno, + loc==0 ? "overwrite" : "new entry")); + assert( pPage->isInit ); + allocateTempSpace(pBt); + newCell = pBt->pTmpSpace; + if( newCell==0 ) return SQLITE_NOMEM; + rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); + if( rc ) goto end_insert; + assert( szNew==cellSizePtr(pPage, newCell) ); + assert( szNew <= MX_CELL_SIZE(pBt) ); + idx = pCur->aiIdx[pCur->iPage]; + if( loc==0 ){ + u16 szOld; + assert( idxnCell ); + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ){ + goto end_insert; } + oldCell = findCell(pPage, idx); + if( !pPage->leaf ){ + memcpy(newCell, oldCell, 4); + } + szOld = cellSizePtr(pPage, oldCell); + rc = clearCell(pPage, oldCell); + dropCell(pPage, idx, szOld, &rc); + if( rc ) goto end_insert; + }else if( loc<0 && pPage->nCell>0 ){ + assert( pPage->leaf ); + idx = ++pCur->aiIdx[pCur->iPage]; + }else{ + assert( pPage->leaf ); } -} - -/* -** Release any memory held by the Mem. This may leave the Mem in an -** inconsistent state, for example with (Mem.z==0) and -** (Mem.type==SQLITE_TEXT). -*/ -SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ - sqlite3VdbeMemReleaseExternal(p); - sqlite3DbFree(p->db, p->zMalloc); - p->z = 0; - p->zMalloc = 0; - p->xDel = 0; -} + insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); -/* -** Convert a 64-bit IEEE double into a 64-bit signed integer. -** If the double is too large, return 0x8000000000000000. -** -** Most systems appear to do this simply by assigning -** variables and without the extra range tests. But -** there are reports that windows throws an expection -** if the floating point value is out of range. (See ticket #2880.) -** Because we do not completely understand the problem, we will -** take the conservative approach and always do range tests -** before attempting the conversion. -*/ -static i64 doubleToInt64(double r){ -#ifdef SQLITE_OMIT_FLOATING_POINT - /* When floating-point is omitted, double and int64 are the same thing */ - return r; -#else - /* - ** Many compilers we encounter do not define constants for the - ** minimum and maximum 64-bit integers, or they define them - ** inconsistently. And many do not understand the "LL" notation. - ** So we define our own static constants here using nothing - ** larger than a 32-bit integer constant. + /* If no error has occured and pPage has an overflow cell, call balance() + ** to redistribute the cells within the tree. Since balance() may move + ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey + ** variables. + ** + ** Previous versions of SQLite called moveToRoot() to move the cursor + ** back to the root page as balance() used to invalidate the contents + ** of BtCursor.apPage[] and BtCursor.aiIdx[]. Instead of doing that, + ** set the cursor state to "invalid". This makes common insert operations + ** slightly faster. + ** + ** There is a subtle but important optimization here too. When inserting + ** multiple records into an intkey b-tree using a single cursor (as can + ** happen while processing an "INSERT INTO ... SELECT" statement), it + ** is advantageous to leave the cursor pointing to the last entry in + ** the b-tree if possible. If the cursor is left pointing to the last + ** entry in the table, and the next row inserted has an integer key + ** larger than the largest existing key, it is possible to insert the + ** row without seeking the cursor. This can be a big performance boost. */ - static const i64 maxInt = LARGEST_INT64; - static const i64 minInt = SMALLEST_INT64; + pCur->info.nSize = 0; + pCur->validNKey = 0; + if( rc==SQLITE_OK && pPage->nOverflow ){ + rc = balance(pCur); - if( r<(double)minInt ){ - return minInt; - }else if( r>(double)maxInt ){ - /* minInt is correct here - not maxInt. It turns out that assigning - ** a very large positive number to an integer results in a very large - ** negative integer. This makes no sense, but it is what x86 hardware - ** does so for compatibility we will do the same in software. */ - return minInt; - }else{ - return (i64)r; + /* Must make sure nOverflow is reset to zero even if the balance() + ** fails. Internal data structure corruption will result otherwise. + ** Also, set the cursor state to invalid. This stops saveCursorPosition() + ** from trying to save the current position of the cursor. */ + pCur->apPage[pCur->iPage]->nOverflow = 0; + pCur->eState = CURSOR_INVALID; } -#endif + assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); + +end_insert: + return rc; } /* -** Return some kind of integer value which is the best we can do -** at representing the value that *pMem describes as an integer. -** If pMem is an integer, then the value is exact. If pMem is -** a floating-point then the value returned is the integer part. -** If pMem is a string or blob, then we make an attempt to convert -** it into a integer and return that. If pMem represents an -** an SQL-NULL value, return 0. -** -** If pMem represents a string value, its encoding might be changed. +** Delete the entry that the cursor is pointing to. The cursor +** is left pointing at a arbitrary location. */ -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ - int flags; - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - flags = pMem->flags; - if( flags & MEM_Int ){ - return pMem->u.i; - }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->r); - }else if( flags & (MEM_Str|MEM_Blob) ){ - i64 value; - assert( pMem->z || pMem->n==0 ); - testcase( pMem->z==0 ); - sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); - return value; - }else{ - return 0; +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + int rc; /* Return code */ + MemPage *pPage; /* Page to delete cell from */ + unsigned char *pCell; /* Pointer to cell to delete */ + int iCellIdx; /* Index of cell to delete */ + int iCellDepth; /* Depth of node containing pCell */ + + assert( cursorHoldsMutex(pCur) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( !pBt->readOnly ); + assert( pCur->wrFlag ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + assert( !hasReadConflicts(p, pCur->pgnoRoot) ); + + if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) + || NEVER(pCur->eState!=CURSOR_VALID) + ){ + return SQLITE_ERROR; /* Something has gone awry. */ } -} -/* -** Return the best representation of pMem that we can get into a -** double. If pMem is already a double or an integer, return its -** value. If it is a string or blob, try to convert it to a double. -** If it is a NULL, return 0.0. -*/ -SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - if( pMem->flags & MEM_Real ){ - return pMem->r; - }else if( pMem->flags & MEM_Int ){ - return (double)pMem->u.i; - }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - double val = (double)0; - sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); - return val; - }else{ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - return (double)0; + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); } -} -/* -** The MEM structure is already a MEM_Real. Try to also make it a -** MEM_Int if we can. -*/ -SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ - assert( pMem->flags & MEM_Real ); - assert( (pMem->flags & MEM_RowSet)==0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + iCellDepth = pCur->iPage; + iCellIdx = pCur->aiIdx[iCellDepth]; + pPage = pCur->apPage[iCellDepth]; + pCell = findCell(pPage, iCellIdx); - pMem->u.i = doubleToInt64(pMem->r); + /* If the page containing the entry to delete is not a leaf page, move + ** the cursor to the largest entry in the tree that is smaller than + ** the entry being deleted. This cell will replace the cell being deleted + ** from the internal node. The 'previous' entry is used for this instead + ** of the 'next' entry, as the previous entry is always a part of the + ** sub-tree headed by the child page of the cell being deleted. This makes + ** balancing the tree following the delete operation easier. */ + if( !pPage->leaf ){ + int notUsed; + rc = sqlite3BtreePrevious(pCur, ¬Used); + if( rc ) return rc; + } - /* Only mark the value as an integer if - ** - ** (1) the round-trip conversion real->int->real is a no-op, and - ** (2) The integer is neither the largest nor the smallest - ** possible integer (ticket #3922) - ** - ** The second and third terms in the following conditional enforces - ** the second condition under the assumption that addition overflow causes - ** values to wrap around. On x86 hardware, the third term is always - ** true and could be omitted. But we leave it in because other - ** architectures might behave differently. + /* Save the positions of any other cursors open on this table before + ** making any modifications. Make the page containing the entry to be + ** deleted writable. Then free any overflow pages associated with the + ** entry and finally remove the cell itself from within the page. */ - if( pMem->r==(double)pMem->u.i && pMem->u.i>SMALLEST_INT64 - && ALWAYS(pMem->u.iflags |= MEM_Int; - } -} + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + rc = clearCell(pPage, pCell); + dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); + if( rc ) return rc; -/* -** Convert pMem to type integer. Invalidate any prior representations. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + /* If the cell deleted was not located on a leaf page, then the cursor + ** is currently pointing to the largest entry in the sub-tree headed + ** by the child-page of the cell that was just deleted from an internal + ** node. The cell from the leaf node needs to be moved to the internal + ** node to replace the deleted cell. */ + if( !pPage->leaf ){ + MemPage *pLeaf = pCur->apPage[pCur->iPage]; + int nCell; + Pgno n = pCur->apPage[iCellDepth+1]->pgno; + unsigned char *pTmp; - pMem->u.i = sqlite3VdbeIntValue(pMem); - MemSetTypeFlag(pMem, MEM_Int); - return SQLITE_OK; -} + pCell = findCell(pLeaf, pLeaf->nCell-1); + nCell = cellSizePtr(pLeaf, pCell); + assert( MX_CELL_SIZE(pBt) >= nCell ); -/* -** Convert pMem so that it is of type MEM_Real. -** Invalidate any prior representations. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + allocateTempSpace(pBt); + pTmp = pBt->pTmpSpace; - pMem->r = sqlite3VdbeRealValue(pMem); - MemSetTypeFlag(pMem, MEM_Real); - return SQLITE_OK; -} + rc = sqlite3PagerWrite(pLeaf->pDbPage); + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); + if( rc ) return rc; + } -/* -** Convert pMem so that it has types MEM_Real or MEM_Int or both. -** Invalidate any prior representations. -** -** Every effort is made to force the conversion, even if the input -** is a string that does not look completely like a number. Convert -** as much of the string as we can and ignore the rest. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ - assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ - MemSetTypeFlag(pMem, MEM_Int); - }else{ - pMem->r = sqlite3VdbeRealValue(pMem); - MemSetTypeFlag(pMem, MEM_Real); - sqlite3VdbeIntegerAffinity(pMem); + /* Balance the tree. If the entry deleted was located on a leaf page, + ** then the cursor still points to that page. In this case the first + ** call to balance() repairs the tree, and the if(...) condition is + ** never true. + ** + ** Otherwise, if the entry deleted was on an internal node page, then + ** pCur is pointing to the leaf page from which a cell was removed to + ** replace the cell deleted from the internal node. This is slightly + ** tricky as the leaf node may be underfull, and the internal node may + ** be either under or overfull. In this case run the balancing algorithm + ** on the leaf node first. If the balance proceeds far enough up the + ** tree that we can be sure that any problem in the internal node has + ** been corrected, so be it. Otherwise, after balancing the leaf node, + ** walk the cursor up the tree to the internal node and balance it as + ** well. */ + rc = balance(pCur); + if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ + while( pCur->iPage>iCellDepth ){ + releasePage(pCur->apPage[pCur->iPage--]); } + rc = balance(pCur); } - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); - pMem->flags &= ~(MEM_Str|MEM_Blob); - return SQLITE_OK; -} -/* -** Delete any previous value and set the value stored in *pMem to NULL. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ - if( pMem->flags & MEM_Frame ){ - VdbeFrame *pFrame = pMem->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; - } - if( pMem->flags & MEM_RowSet ){ - sqlite3RowSetClear(pMem->u.pRowSet); + if( rc==SQLITE_OK ){ + moveToRoot(pCur); } - MemSetTypeFlag(pMem, MEM_Null); - pMem->type = SQLITE_NULL; + return rc; } /* -** Delete any previous value and set the value to be a BLOB of length -** n containing all zeros. +** Create a new BTree table. Write into *piTable the page +** number for the root page of the new table. +** +** The type of type is determined by the flags parameter. Only the +** following values of flags are currently in use. Other values for +** flags might not work: +** +** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys +** BTREE_ZERODATA Used for SQL indices */ -SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ - sqlite3VdbeMemRelease(pMem); - pMem->flags = MEM_Blob|MEM_Zero; - pMem->type = SQLITE_BLOB; - pMem->n = 0; - if( n<0 ) n = 0; - pMem->u.nZero = n; - pMem->enc = SQLITE_UTF8; +static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ + BtShared *pBt = p->pBt; + MemPage *pRoot; + Pgno pgnoRoot; + int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ -#ifdef SQLITE_OMIT_INCRBLOB - sqlite3VdbeMemGrow(pMem, n, 0); - if( pMem->z ){ - pMem->n = n; - memset(pMem->z, 0, n); + assert( sqlite3BtreeHoldsMutex(p) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( !pBt->readOnly ); + +#ifdef SQLITE_OMIT_AUTOVACUUM + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ){ + return rc; } -#endif -} +#else + if( pBt->autoVacuum ){ + Pgno pgnoMove; /* Move a page here to make room for the root-page */ + MemPage *pPageMove; /* The page to move to. */ -/* -** Delete any previous value and set the value stored in *pMem to val, -** manifest type INTEGER. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ - sqlite3VdbeMemRelease(pMem); - pMem->u.i = val; - pMem->flags = MEM_Int; - pMem->type = SQLITE_INTEGER; -} + /* Creating a new table may probably require moving an existing database + ** to make room for the new tables root page. In case this page turns + ** out to be an overflow page, delete all overflow page-map caches + ** held by open cursors. + */ + invalidateAllOverflowCache(pBt); -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** Delete any previous value and set the value stored in *pMem to val, -** manifest type REAL. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ - if( sqlite3IsNaN(val) ){ - sqlite3VdbeMemSetNull(pMem); - }else{ - sqlite3VdbeMemRelease(pMem); - pMem->r = val; - pMem->flags = MEM_Real; - pMem->type = SQLITE_FLOAT; - } -} -#endif + /* Read the value of meta[3] from the database to determine where the + ** root page of the new table should go. meta[3] is the largest root-page + ** created so far, so the new root-page is (meta[3]+1). + */ + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); + pgnoRoot++; -/* -** Delete any previous value and set the value of pMem to be an -** empty boolean index. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ - sqlite3 *db = pMem->db; - assert( db!=0 ); - assert( (pMem->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRaw(db, 64); - if( db->mallocFailed ){ - pMem->flags = MEM_Null; - }else{ - assert( pMem->zMalloc ); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, - sqlite3DbMallocSize(db, pMem->zMalloc)); - assert( pMem->u.pRowSet!=0 ); - pMem->flags = MEM_RowSet; - } -} + /* The new root-page may not be allocated on a pointer-map page, or the + ** PENDING_BYTE page. + */ + while( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || + pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ + pgnoRoot++; + } + assert( pgnoRoot>=3 ); -/* -** Return true if the Mem object contains a TEXT or BLOB that is -** too large - whose size exceeds SQLITE_MAX_LENGTH. -*/ -SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ - assert( p->db!=0 ); - if( p->flags & (MEM_Str|MEM_Blob) ){ - int n = p->n; - if( p->flags & MEM_Zero ){ - n += p->u.nZero; + /* Allocate a page. The page that currently resides at pgnoRoot will + ** be moved to the allocated page (unless the allocated page happens + ** to reside at pgnoRoot). + */ + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); + if( rc!=SQLITE_OK ){ + return rc; } - return n>p->db->aLimit[SQLITE_LIMIT_LENGTH]; - } - return 0; -} -#ifdef SQLITE_DEBUG -/* -** This routine prepares a memory cell for modication by breaking -** its link to a shallow copy and by marking any current shallow -** copies of this cell as invalid. -** -** This is used for testing and debugging only - to make sure shallow -** copies are not misused. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ - int i; - Mem *pX; - for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ - if( pX->pScopyFrom==pMem ){ - pX->flags |= MEM_Invalid; - pX->pScopyFrom = 0; + if( pgnoMove!=pgnoRoot ){ + /* pgnoRoot is the page that will be used for the root-page of + ** the new table (assuming an error did not occur). But we were + ** allocated pgnoMove. If required (i.e. if it was not allocated + ** by extending the file), the current page at position pgnoMove + ** is already journaled. + */ + u8 eType = 0; + Pgno iPtrPage = 0; + + releasePage(pPageMove); + + /* Move the page currently at pgnoRoot to pgnoMove. */ + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); + if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ + rc = SQLITE_CORRUPT_BKPT; + } + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + assert( eType!=PTRMAP_ROOTPAGE ); + assert( eType!=PTRMAP_FREEPAGE ); + rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove, 0); + releasePage(pRoot); + + /* Obtain the page at pgnoRoot */ + if( rc!=SQLITE_OK ){ + return rc; + } + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + }else{ + pRoot = pPageMove; + } + + /* Update the pointer-map and meta-data with the new root-page number. */ + ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); + if( rc ){ + releasePage(pRoot); + return rc; } - } - pMem->pScopyFrom = 0; -} -#endif /* SQLITE_DEBUG */ -/* -** Size of struct Mem not including the Mem.zMalloc member. -*/ -#define MEMCELLSIZE (size_t)(&(((Mem *)0)->zMalloc)) + /* When the new root page was allocated, page 1 was made writable in + ** order either to increase the database filesize, or to decrement the + ** freelist count. Hence, the sqlite3BtreeUpdateMeta() call cannot fail. + */ + assert( sqlite3PagerIswriteable(pBt->pPage1->pDbPage) ); + rc = sqlite3BtreeUpdateMeta(p, 4, pgnoRoot); + if( NEVER(rc) ){ + releasePage(pRoot); + return rc; + } -/* -** Make an shallow copy of pFrom into pTo. Prior contents of -** pTo are freed. The pFrom->z field is not duplicated. If -** pFrom->z is used, then pTo->z points to the same thing as pFrom->z -** and flags gets srcType (either MEM_Ephem or MEM_Static). -*/ -SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ - assert( (pFrom->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemReleaseExternal(pTo); - memcpy(pTo, pFrom, MEMCELLSIZE); - pTo->xDel = 0; - if( (pFrom->flags&MEM_Static)==0 ){ - pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); - assert( srcType==MEM_Ephem || srcType==MEM_Static ); - pTo->flags |= srcType; + }else{ + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ) return rc; } +#endif + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); + sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); + *piTable = (int)pgnoRoot; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCreateTable(p, piTable, flags); + sqlite3BtreeLeave(p); + return rc; } /* -** Make a full copy of pFrom into pTo. Prior contents of pTo are -** freed before the copy is made. +** Erase the given database page and all its children. Return +** the page to the freelist. */ -SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ - int rc = SQLITE_OK; +static int clearDatabasePage( + BtShared *pBt, /* The BTree that contains the table */ + Pgno pgno, /* Page number to clear */ + int freePageFlag, /* Deallocate page if true */ + int *pnChange /* Add number of Cells freed to this counter */ +){ + MemPage *pPage; + int rc; + unsigned char *pCell; + int i; - assert( (pFrom->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemReleaseExternal(pTo); - memcpy(pTo, pFrom, MEMCELLSIZE); - pTo->flags &= ~MEM_Dyn; + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } - if( pTo->flags&(MEM_Str|MEM_Blob) ){ - if( 0==(pFrom->flags&MEM_Static) ){ - pTo->flags |= MEM_Ephem; - rc = sqlite3VdbeMemMakeWriteable(pTo); + rc = getAndInitPage(pBt, pgno, &pPage); + if( rc ) return rc; + for(i=0; inCell; i++){ + pCell = findCell(pPage, i); + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); + if( rc ) goto cleardatabasepage_out; } + rc = clearCell(pPage, pCell); + if( rc ) goto cleardatabasepage_out; + } + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + }else if( pnChange ){ + assert( pPage->intKey ); + *pnChange += pPage->nCell; + } + if( freePageFlag ){ + freePage(pPage, &rc); + }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ + zeroPage(pPage, pPage->aData[0] | PTF_LEAF); } +cleardatabasepage_out: + releasePage(pPage); return rc; } /* -** Transfer the contents of pFrom to pTo. Any existing value in pTo is -** freed. If pFrom contains ephemeral data, a copy is made. +** Delete all information from a single table in the database. iTable is +** the page number of the root of the table. After this routine returns, +** the root page is empty, but still exists. ** -** pFrom contains an SQL NULL when this routine returns. +** This routine will fail with SQLITE_LOCKED if there are any open +** read cursors on the table. Open write cursors are moved to the +** root of the table. +** +** If pnChange is not NULL, then table iTable must be an intkey table. The +** integer value pointed to by pnChange is incremented by the number of +** entries in the table. */ -SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ - assert( pFrom->db==0 || sqlite3_mutex_held(pFrom->db->mutex) ); - assert( pTo->db==0 || sqlite3_mutex_held(pTo->db->mutex) ); - assert( pFrom->db==0 || pTo->db==0 || pFrom->db==pTo->db ); +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); - sqlite3VdbeMemRelease(pTo); - memcpy(pTo, pFrom, sizeof(Mem)); - pFrom->flags = MEM_Null; - pFrom->xDel = 0; - pFrom->zMalloc = 0; + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); + + rc = saveAllCursors(pBt, (Pgno)iTable, 0); + if( SQLITE_OK==rc ){ + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + } + sqlite3BtreeLeave(p); + return rc; } /* -** Change the value of a Mem to be a string or a BLOB. +** Erase all information in a table and add the root of the table to +** the freelist. Except, the root of the principle table (the one on +** page 1) is never added to the freelist. ** -** The memory management strategy depends on the value of the xDel -** parameter. If the value passed is SQLITE_TRANSIENT, then the -** string is copied into a (possibly existing) buffer managed by the -** Mem structure. Otherwise, any existing buffer is freed and the -** pointer copied. +** This routine will fail with SQLITE_LOCKED if there are any open +** cursors on the table. ** -** If the string is too large (if it exceeds the SQLITE_LIMIT_LENGTH -** size limit) then no memory allocation occurs. If the string can be -** stored without allocating memory, then it is. If a memory allocation -** is required to store the string, then value of pMem is unchanged. In -** either case, SQLITE_TOOBIG is returned. +** If AUTOVACUUM is enabled and the page at iTable is not the last +** root page in the database file, then the last root page +** in the database file is moved into the slot formerly occupied by +** iTable and that last slot formerly occupied by the last root page +** is added to the freelist instead of iTable. In this say, all +** root pages are kept at the beginning of the database file, which +** is necessary for AUTOVACUUM to work right. *piMoved is set to the +** page number that used to be the last root page in the file before +** the move. If no page gets moved, *piMoved is set to 0. +** The last root page is recorded in meta[3] and the value of +** meta[3] is updated by this procedure. */ -SQLITE_PRIVATE int sqlite3VdbeMemSetStr( - Mem *pMem, /* Memory cell to set to string value */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*) /* Destructor function */ -){ - int nByte = n; /* New value for pMem->n */ - int iLimit; /* Maximum allowed string or blob size */ - u16 flags = 0; /* New value for pMem->flags */ +static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ + int rc; + MemPage *pPage = 0; + BtShared *pBt = p->pBt; - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->inTrans==TRANS_WRITE ); - /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ - if( !z ){ - sqlite3VdbeMemSetNull(pMem); - return SQLITE_OK; + /* It is illegal to drop a table if any cursors are open on the + ** database. This is because in auto-vacuum mode the backend may + ** need to move another root-page to fill a gap left by the deleted + ** root page. If an open cursor was using this page a problem would + ** occur. + ** + ** This error is caught long before control reaches this point. + */ + if( NEVER(pBt->pCursor) ){ + sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db); + return SQLITE_LOCKED_SHAREDCACHE; } - if( pMem->db ){ - iLimit = pMem->db->aLimit[SQLITE_LIMIT_LENGTH]; - }else{ - iLimit = SQLITE_MAX_LENGTH; - } - flags = (enc==0?MEM_Blob:MEM_Str); - if( nByte<0 ){ - assert( enc!=0 ); - if( enc==SQLITE_UTF8 ){ - for(nByte=0; nByte<=iLimit && z[nByte]; nByte++){} - }else{ - for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} - } - flags |= MEM_Term; + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + if( rc ) return rc; + rc = sqlite3BtreeClearTable(p, iTable, 0); + if( rc ){ + releasePage(pPage); + return rc; } - /* The following block sets the new values of Mem.z and Mem.xDel. It - ** also sets a flag in local variable "flags" to indicate the memory - ** management (one of MEM_Dyn or MEM_Static). - */ - if( xDel==SQLITE_TRANSIENT ){ - int nAlloc = nByte; - if( flags&MEM_Term ){ - nAlloc += (enc==SQLITE_UTF8?1:2); - } - if( nByte>iLimit ){ - return SQLITE_TOOBIG; - } - if( sqlite3VdbeMemGrow(pMem, nAlloc, 0) ){ - return SQLITE_NOMEM; - } - memcpy(pMem->z, z, nAlloc); - }else if( xDel==SQLITE_DYNAMIC ){ - sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = pMem->z = (char *)z; - pMem->xDel = 0; - }else{ - sqlite3VdbeMemRelease(pMem); - pMem->z = (char *)z; - pMem->xDel = xDel; - flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); - } + *piMoved = 0; - pMem->n = nByte; - pMem->flags = flags; - pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); - pMem->type = (enc==0 ? SQLITE_BLOB : SQLITE_TEXT); + if( iTable>1 ){ +#ifdef SQLITE_OMIT_AUTOVACUUM + freePage(pPage, &rc); + releasePage(pPage); +#else + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); -#ifndef SQLITE_OMIT_UTF16 - if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ - return SQLITE_NOMEM; - } -#endif + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. + */ + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; + } - if( nByte>iLimit ){ - return SQLITE_TOOBIG; - } + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. + */ + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); - return SQLITE_OK; + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); + releasePage(pPage); + } +#endif + }else{ + /* If sqlite3BtreeDropTable was called on page 1. + ** This really never should happen except in a corrupt + ** database. + */ + zeroPage(pPage, PTF_INTKEY|PTF_LEAF ); + releasePage(pPage); + } + return rc; } +SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeDropTable(p, iTable, piMoved); + sqlite3BtreeLeave(p); + return rc; +} + /* -** Compare the values contained by the two memory cells, returning -** negative, zero or positive if pMem1 is less than, equal to, or greater -** than pMem2. Sorting order is NULL's first, followed by numbers (integers -** and reals) sorted numerically, followed by text ordered by the collating -** sequence pColl and finally blob's ordered by memcmp(). +** This function may only be called if the b-tree connection already +** has a read or write transaction open on the database. ** -** Two NULL values are considered equal by this function. +** Read the meta-information out of a database file. Meta[0] +** is the number of free pages currently in the database. Meta[1] +** through meta[15] are available for use by higher layers. Meta[0] +** is read-only, the others are read/write. +** +** The schema layer numbers meta values differently. At the schema +** layer (and the SetCookie and ReadCookie opcodes) the number of +** free pages is not visible. So Cookie[0] is the same as Meta[1]. */ -SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ - int rc; - int f1, f2; - int combined_flags; - - f1 = pMem1->flags; - f2 = pMem2->flags; - combined_flags = f1|f2; - assert( (combined_flags & MEM_RowSet)==0 ); - - /* If one value is NULL, it is less than the other. If both values - ** are NULL, return 0. - */ - if( combined_flags&MEM_Null ){ - return (f2&MEM_Null) - (f1&MEM_Null); - } +SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ + BtShared *pBt = p->pBt; - /* If one value is a number and the other is not, the number is less. - ** If both are numbers, compare as reals if one is a real, or as integers - ** if both values are integers. - */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - if( !(f1&(MEM_Int|MEM_Real)) ){ - return 1; - } - if( !(f2&(MEM_Int|MEM_Real)) ){ - return -1; - } - if( (f1 & f2 & MEM_Int)==0 ){ - double r1, r2; - if( (f1&MEM_Real)==0 ){ - r1 = (double)pMem1->u.i; - }else{ - r1 = pMem1->r; - } - if( (f2&MEM_Real)==0 ){ - r2 = (double)pMem2->u.i; - }else{ - r2 = pMem2->r; - } - if( r1r2 ) return 1; - return 0; - }else{ - assert( f1&MEM_Int ); - assert( f2&MEM_Int ); - if( pMem1->u.i < pMem2->u.i ) return -1; - if( pMem1->u.i > pMem2->u.i ) return 1; - return 0; - } - } + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE ); + assert( SQLITE_OK==querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK) ); + assert( pBt->pPage1 ); + assert( idx>=0 && idx<=15 ); - /* If one value is a string and the other is a blob, the string is less. - ** If both are strings, compare using the collating functions. - */ - if( combined_flags&MEM_Str ){ - if( (f1 & MEM_Str)==0 ){ - return 1; - } - if( (f2 & MEM_Str)==0 ){ - return -1; - } + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); - assert( pMem1->enc==pMem2->enc ); - assert( pMem1->enc==SQLITE_UTF8 || - pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); + /* If auto-vacuum is disabled in this build and this is an auto-vacuum + ** database, mark the database as read-only. */ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ) pBt->readOnly = 1; +#endif - /* The collation sequence must be defined at this point, even if - ** the user deletes the collation sequence after the vdbe program is - ** compiled (this was not always the case). - */ - assert( !pColl || pColl->xCmp ); + sqlite3BtreeLeave(p); +} - if( pColl ){ - if( pMem1->enc==pColl->enc ){ - /* The strings are already in the correct encoding. Call the - ** comparison function directly */ - return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); - }else{ - const void *v1, *v2; - int n1, n2; - Mem c1; - Mem c2; - memset(&c1, 0, sizeof(c1)); - memset(&c2, 0, sizeof(c2)); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - n1 = v1==0 ? 0 : c1.n; - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - n2 = v2==0 ? 0 : c2.n; - rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); - sqlite3VdbeMemRelease(&c1); - sqlite3VdbeMemRelease(&c2); - return rc; - } +/* +** Write meta-information back into the database. Meta[0] is +** read-only and may not be written. +*/ +SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ + BtShared *pBt = p->pBt; + unsigned char *pP1; + int rc; + assert( idx>=1 && idx<=15 ); + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1!=0 ); + pP1 = pBt->pPage1->aData; + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pP1[36 + idx*4], iMeta); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_INCR_VACUUM ){ + assert( pBt->autoVacuum || iMeta==0 ); + assert( iMeta==0 || iMeta==1 ); + pBt->incrVacuum = (u8)iMeta; } - /* If a NULL pointer was passed as the collate function, fall through - ** to the blob case and use memcmp(). */ - } - - /* Both values must be blobs. Compare using memcmp(). */ - rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); - if( rc==0 ){ - rc = pMem1->n - pMem2->n; +#endif } + sqlite3BtreeLeave(p); return rc; } +#ifndef SQLITE_OMIT_BTREECOUNT /* -** Move data out of a btree key or data field and into a Mem structure. -** The data or key is taken from the entry that pCur is currently pointing -** to. offset and amt determine what portion of the data or key to retrieve. -** key is true to get the key or false to get data. The result is written -** into the pMem element. -** -** The pMem structure is assumed to be uninitialized. Any prior content -** is overwritten without being freed. +** The first argument, pCur, is a cursor opened on some b-tree. Count the +** number of entries in the b-tree and write the result to *pnEntry. ** -** If this routine fails for any reason (malloc returns NULL or unable -** to read from the disk) then the pMem is left in an inconsistent state. +** SQLITE_OK is returned if the operation is successfully executed. +** Otherwise, if an error is encountered (i.e. an IO error or database +** corruption) an SQLite error code is returned. */ -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( - BtCursor *pCur, /* Cursor pointing at record to retrieve. */ - int offset, /* Offset from the start of data to return bytes from. */ - int amt, /* Number of bytes to return. */ - int key, /* If true, retrieve from the btree key, not data. */ - Mem *pMem /* OUT: Return data in this Mem structure. */ -){ - char *zData; /* Data from the btree layer */ - int available = 0; /* Number of bytes available on the local btree page */ - int rc = SQLITE_OK; /* Return code */ +SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ + i64 nEntry = 0; /* Value to return in *pnEntry */ + int rc; /* Return code */ + rc = moveToRoot(pCur); - assert( sqlite3BtreeCursorIsValid(pCur) ); + /* Unless an error occurs, the following loop runs one iteration for each + ** page in the B-Tree structure (not including overflow pages). + */ + while( rc==SQLITE_OK ){ + int iIdx; /* Index of child node in parent */ + MemPage *pPage; /* Current page of the b-tree */ - /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() - ** that both the BtShared and database handle mutexes are held. */ - assert( (pMem->flags & MEM_RowSet)==0 ); - if( key ){ - zData = (char *)sqlite3BtreeKeyFetch(pCur, &available); - }else{ - zData = (char *)sqlite3BtreeDataFetch(pCur, &available); - } - assert( zData!=0 ); + /* If this is a leaf page or the tree is not an int-key tree, then + ** this page contains countable entries. Increment the entry counter + ** accordingly. + */ + pPage = pCur->apPage[pCur->iPage]; + if( pPage->leaf || !pPage->intKey ){ + nEntry += pPage->nCell; + } - if( offset+amt<=available && (pMem->flags&MEM_Dyn)==0 ){ - sqlite3VdbeMemRelease(pMem); - pMem->z = &zData[offset]; - pMem->flags = MEM_Blob|MEM_Ephem; - }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ - pMem->flags = MEM_Blob|MEM_Dyn|MEM_Term; - pMem->enc = 0; - pMem->type = SQLITE_BLOB; - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + /* pPage is a leaf node. This loop navigates the cursor so that it + ** points to the first interior cell that it points to the parent of + ** the next page in the tree that has not yet been visited. The + ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell + ** of the page, or to the number of cells in the page if the next page + ** to visit is the right-child of its parent. + ** + ** If all pages in the tree have been visited, return SQLITE_OK to the + ** caller. + */ + if( pPage->leaf ){ + do { + if( pCur->iPage==0 ){ + /* All pages of the b-tree have been visited. Return successfully. */ + *pnEntry = nEntry; + return SQLITE_OK; + } + moveToParent(pCur); + }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); + + pCur->aiIdx[pCur->iPage]++; + pPage = pCur->apPage[pCur->iPage]; } - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - if( rc!=SQLITE_OK ){ - sqlite3VdbeMemRelease(pMem); + + /* Descend to the child node of the cell that the cursor currently + ** points at. This is the right-child if (iIdx==pPage->nCell). + */ + iIdx = pCur->aiIdx[pCur->iPage]; + if( iIdx==pPage->nCell ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + }else{ + rc = moveToChild(pCur, get4byte(findCell(pPage, iIdx))); } } - pMem->n = amt; + /* An error has occurred. Return an error code. */ return rc; } +#endif -/* This function is only available internally, it is not part of the -** external API. It works in a similar way to sqlite3_value_text(), -** except the data returned is in the encoding specified by the second -** parameter, which must be one of SQLITE_UTF16BE, SQLITE_UTF16LE or -** SQLITE_UTF8. -** -** (2006-02-16:) The enc value can be or-ed with SQLITE_UTF16_ALIGNED. -** If that is the case, then the result must be aligned on an even byte -** boundary. +/* +** Return the pager associated with a BTree. This routine is used for +** testing and debugging only. */ -SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ - if( !pVal ) return 0; - - assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); - assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); +SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ + return p->pBt->pPager; +} - if( pVal->flags&MEM_Null ){ - return 0; +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Append a message to the error message string. +*/ +static void checkAppendMsg( + IntegrityCk *pCheck, + char *zMsg1, + const char *zFormat, + ... +){ + va_list ap; + if( !pCheck->mxErr ) return; + pCheck->mxErr--; + pCheck->nErr++; + va_start(ap, zFormat); + if( pCheck->errMsg.nChar ){ + sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } - assert( (MEM_Blob>>3) == MEM_Str ); - pVal->flags |= (pVal->flags & MEM_Blob)>>3; - expandBlob(pVal); - if( pVal->flags&MEM_Str ){ - sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); - if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ - assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); - if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ - return 0; - } - } - sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-59893-45467 */ - }else{ - assert( (pVal->flags&MEM_Blob)==0 ); - sqlite3VdbeMemStringify(pVal, enc); - assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + if( zMsg1 ){ + sqlite3StrAccumAppend(&pCheck->errMsg, zMsg1, -1); } - assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 - || pVal->db->mallocFailed ); - if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - return pVal->z; - }else{ - return 0; + sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); + va_end(ap); + if( pCheck->errMsg.mallocFailed ){ + pCheck->mallocFailed = 1; } } +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ +#ifndef SQLITE_OMIT_INTEGRITY_CHECK /* -** Create a new sqlite3_value object. +** Add 1 to the reference count for page iPage. If this is the second +** reference to the page, add an error message to pCheck->zErrMsg. +** Return 1 if there are 2 ore more references to the page and 0 if +** if this is the first reference to the page. +** +** Also check that the page number is in bounds. */ -SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ - Mem *p = sqlite3DbMallocZero(db, sizeof(*p)); - if( p ){ - p->flags = MEM_Null; - p->type = SQLITE_NULL; - p->db = db; +static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ + if( iPage==0 ) return 1; + if( iPage>pCheck->nPage ){ + checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); + return 1; } - return p; + if( pCheck->anRef[iPage]==1 ){ + checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); + return 1; + } + return (pCheck->anRef[iPage]++)>1; } +#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Create a new sqlite3_value object, containing the value of pExpr. -** -** This only works for very simple expressions that consist of one constant -** token (i.e. "5", "5.1", "'a string'"). If the expression can -** be converted directly into a value, then the value is allocated and -** a pointer written to *ppVal. The caller is responsible for deallocating -** the value by passing it to sqlite3ValueFree() later on. If the expression -** cannot be converted to a value, then *ppVal is set to NULL. +** Check that the entry in the pointer-map for page iChild maps to +** page iParent, pointer type ptrType. If not, append an error message +** to pCheck. */ -SQLITE_PRIVATE int sqlite3ValueFromExpr( - sqlite3 *db, /* The database connection */ - Expr *pExpr, /* The expression to evaluate */ - u8 enc, /* Encoding to use */ - u8 affinity, /* Affinity to use */ - sqlite3_value **ppVal /* Write the new value here */ +static void checkPtrmap( + IntegrityCk *pCheck, /* Integrity check context */ + Pgno iChild, /* Child page number */ + u8 eType, /* Expected pointer map type */ + Pgno iParent, /* Expected pointer map parent page number */ + char *zContext /* Context description (used for error msg) */ ){ - int op; - char *zVal = 0; - sqlite3_value *pVal = 0; - int negInt = 1; - const char *zNeg = ""; + int rc; + u8 ePtrmapType; + Pgno iPtrmapParent; - if( !pExpr ){ - *ppVal = 0; - return SQLITE_OK; + rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; + checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); + return; } - op = pExpr->op; - /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2. - ** The ifdef here is to enable us to achieve 100% branch test coverage even - ** when SQLITE_ENABLE_STAT2 is omitted. - */ -#ifdef SQLITE_ENABLE_STAT2 - if( op==TK_REGISTER ) op = pExpr->op2; -#else - if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; -#endif - - /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ + checkAppendMsg(pCheck, zContext, + "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + iChild, eType, iParent, ePtrmapType, iPtrmapParent); } +} +#endif - if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ - pVal = sqlite3ValueNew(db); - if( pVal==0 ) goto no_mem; - if( ExprHasProperty(pExpr, EP_IntValue) ){ - sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); - }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); - if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT; +/* +** Check the integrity of the freelist or of an overflow page list. +** Verify that the number of pages on the list is N. +*/ +static void checkList( + IntegrityCk *pCheck, /* Integrity checking context */ + int isFreeList, /* True for a freelist. False for overflow page list */ + int iPage, /* Page number for first page in the list */ + int N, /* Expected number of pages in the list */ + char *zContext /* Context for error messages */ +){ + int i; + int expected = N; + int iFirst = iPage; + while( N-- > 0 && pCheck->mxErr ){ + DbPage *pOvflPage; + unsigned char *pOvflData; + if( iPage<1 ){ + checkAppendMsg(pCheck, zContext, + "%d of %d pages missing from overflow list starting at %d", + N+1, expected, iFirst); + break; } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); - }else{ - sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + if( checkRef(pCheck, iPage, zContext) ) break; + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ + checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); + break; } - if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; - if( enc!=SQLITE_UTF8 ){ - sqlite3VdbeChangeEncoding(pVal, enc); + pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); + if( isFreeList ){ + int n = get4byte(&pOvflData[4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + } +#endif + if( n>(int)pCheck->pBt->usableSize/4-2 ){ + checkAppendMsg(pCheck, zContext, + "freelist leaf count too big on page %d", iPage); + N--; + }else{ + for(i=0; ipBt->autoVacuum ){ + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + } +#endif + checkRef(pCheck, iFreePage, zContext); + } + N -= n; + } } - }else if( op==TK_UMINUS ) { - /* This branch happens for multiple negative signs. Ex: -(-5) */ - if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){ - sqlite3VdbeMemNumerify(pVal); - pVal->u.i = -1 * pVal->u.i; - /* (double)-1 In case of SQLITE_OMIT_FLOATING_POINT... */ - pVal->r = (double)-1 * pVal->r; - sqlite3ValueApplyAffinity(pVal, affinity, enc); +#ifndef SQLITE_OMIT_AUTOVACUUM + else{ + /* If this database supports auto-vacuum and iPage is not the last + ** page in this overflow list, check that the pointer-map entry for + ** the following page matches iPage. + */ + if( pCheck->pBt->autoVacuum && N>0 ){ + i = get4byte(pOvflData); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + } } +#endif + iPage = get4byte(pOvflData); + sqlite3PagerUnref(pOvflPage); } -#ifndef SQLITE_OMIT_BLOB_LITERAL - else if( op==TK_BLOB ){ - int nVal; - assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); - assert( pExpr->u.zToken[1]=='\'' ); - pVal = sqlite3ValueNew(db); - if( !pVal ) goto no_mem; - zVal = &pExpr->u.zToken[2]; - nVal = sqlite3Strlen30(zVal)-1; - assert( zVal[nVal]=='\'' ); - sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2, - 0, SQLITE_DYNAMIC); - } -#endif - - if( pVal ){ - sqlite3VdbeMemStoreType(pVal); - } - *ppVal = pVal; - return SQLITE_OK; - -no_mem: - db->mallocFailed = 1; - sqlite3DbFree(db, zVal); - sqlite3ValueFree(pVal); - *ppVal = 0; - return SQLITE_NOMEM; } +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ +#ifndef SQLITE_OMIT_INTEGRITY_CHECK /* -** Change the string value of an sqlite3_value object +** Do various sanity checks on a single page of a tree. Return +** the tree depth. Root pages return 0. Parents of root pages +** return 1, and so forth. +** +** These checks are done: +** +** 1. Make sure that cells and freeblocks do not overlap +** but combine to completely cover the page. +** NO 2. Make sure cell keys are in order. +** NO 3. Make sure no key is less than or equal to zLowerBound. +** NO 4. Make sure no key is greater than or equal to zUpperBound. +** 5. Check the integrity of overflow pages. +** 6. Recursively call checkTreePage on all children. +** 7. Verify that the depth of all children is the same. +** 8. Make sure this page is at least 33% full or else it is +** the root of the tree. */ -SQLITE_PRIVATE void sqlite3ValueSetStr( - sqlite3_value *v, /* Value to be set */ - int n, /* Length of string z */ - const void *z, /* Text of the new string */ - u8 enc, /* Encoding to use */ - void (*xDel)(void*) /* Destructor for the string */ +static int checkTreePage( + IntegrityCk *pCheck, /* Context for the sanity check */ + int iPage, /* Page number of the page to check */ + char *zParentContext, /* Parent context */ + i64 *pnParentMinKey, + i64 *pnParentMaxKey ){ - if( v ) sqlite3VdbeMemSetStr((Mem *)v, z, n, enc, xDel); -} + MemPage *pPage; + int i, rc, depth, d2, pgno, cnt; + int hdr, cellStart; + int nCell; + u8 *data; + BtShared *pBt; + int usableSize; + char zContext[100]; + char *hit = 0; + i64 nMinKey = 0; + i64 nMaxKey = 0; -/* -** Free an sqlite3_value object -*/ -SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ - if( !v ) return; - sqlite3VdbeMemRelease((Mem *)v); - sqlite3DbFree(((Mem*)v)->db, v); -} + sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); -/* -** Return the number of bytes in the sqlite3_value object assuming -** that it uses the encoding "enc" -*/ -SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ - Mem *p = (Mem*)pVal; - if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ - if( p->flags & MEM_Zero ){ - return p->n + p->u.nZero; - }else{ - return p->n; + /* Check that the page exists + */ + pBt = pCheck->pBt; + usableSize = pBt->usableSize; + if( iPage==0 ) return 0; + if( checkRef(pCheck, iPage, zParentContext) ) return 0; + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + checkAppendMsg(pCheck, zContext, + "unable to get the page. error code=%d", rc); + return 0; + } + + /* Clear MemPage.isInit to make sure the corruption detection code in + ** btreeInitPage() is executed. */ + pPage->isInit = 0; + if( (rc = btreeInitPage(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ + checkAppendMsg(pCheck, zContext, + "btreeInitPage() returns error code %d", rc); + releasePage(pPage); + return 0; + } + + /* Check out all the cells. + */ + depth = 0; + for(i=0; inCell && pCheck->mxErr; i++){ + u8 *pCell; + u32 sz; + CellInfo info; + + /* Check payload overflow pages + */ + sqlite3_snprintf(sizeof(zContext), zContext, + "On tree page %d cell %d: ", iPage, i); + pCell = findCell(pPage,i); + btreeParseCellPtr(pPage, pCell, &info); + sz = info.nData; + if( !pPage->intKey ) sz += (int)info.nKey; + /* For intKey pages, check that the keys are in order. + */ + else if( i==0 ) nMinKey = nMaxKey = info.nKey; + else{ + if( info.nKey <= nMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + } + nMaxKey = info.nKey; + } + assert( sz==info.nPayload ); + if( (sz>info.nLocal) + && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) + ){ + int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); + Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); + } +#endif + checkList(pCheck, 0, pgnoOvfl, nPage, zContext); + } + + /* Check sanity of left child page. + */ + if( !pPage->leaf ){ + pgno = get4byte(pCell); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + } +#endif + d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); + if( i>0 && d2!=depth ){ + checkAppendMsg(pCheck, zContext, "Child page depth differs"); + } + depth = d2; } } - return 0; + + if( !pPage->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + sqlite3_snprintf(sizeof(zContext), zContext, + "On page %d at right child: ", iPage); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + } +#endif + checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); + } + + /* For intKey leaf pages, check that the min/max keys are in order + ** with any left/parent/right pages. + */ + if( pPage->leaf && pPage->intKey ){ + /* if we are a left child page */ + if( pnParentMinKey ){ + /* if we are the left most child page */ + if( !pnParentMaxKey ){ + if( nMaxKey > *pnParentMinKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (max larger than parent min of %lld)", + nMaxKey, *pnParentMinKey); + } + }else{ + if( nMinKey <= *pnParentMinKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (min less than parent min of %lld)", + nMinKey, *pnParentMinKey); + } + if( nMaxKey > *pnParentMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (max larger than parent max of %lld)", + nMaxKey, *pnParentMaxKey); + } + *pnParentMinKey = nMaxKey; + } + /* else if we're a right child page */ + } else if( pnParentMaxKey ){ + if( nMinKey <= *pnParentMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (min less than parent max of %lld)", + nMinKey, *pnParentMaxKey); + } + } + } + + /* Check for complete coverage of the page + */ + data = pPage->aData; + hdr = pPage->hdrOffset; + hit = sqlite3PageMalloc( pBt->pageSize ); + if( hit==0 ){ + pCheck->mallocFailed = 1; + }else{ + int contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + memset(hit+contentOffset, 0, usableSize-contentOffset); + memset(hit, 1, contentOffset); + nCell = get2byte(&data[hdr+3]); + cellStart = hdr + 12 - 4*pPage->leaf; + for(i=0; i=usableSize ){ + checkAppendMsg(pCheck, 0, + "Corruption detected in cell %d on page %d",i,iPage); + }else{ + for(j=pc+size-1; j>=pc; j--) hit[j]++; + } + } + i = get2byte(&data[hdr+1]); + while( i>0 ){ + int size, j; + assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + size = get2byte(&data[i+2]); + assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ + for(j=i+size-1; j>=i; j--) hit[j]++; + j = get2byte(&data[i]); + assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ + assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + i = j; + } + for(i=cnt=0; i1 ){ + checkAppendMsg(pCheck, 0, + "Multiple uses for byte %d of page %d", i, iPage); + break; + } + } + if( cnt!=data[hdr+7] ){ + checkAppendMsg(pCheck, 0, + "Fragmentation of %d bytes reported as %d on page %d", + cnt, data[hdr+7], iPage); + } + } + sqlite3PageFree(hit); + releasePage(pPage); + return depth+1; } +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ -/************** End of vdbemem.c *********************************************/ -/************** Begin file vdbeaux.c *****************************************/ +#ifndef SQLITE_OMIT_INTEGRITY_CHECK /* -** 2003 September 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** This routine does a complete check of the given BTree file. aRoot[] is +** an array of pages numbers were each page number is the root page of +** a table. nRoot is the number of entries in aRoot. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** A read-only or read-write transaction must be opened before calling +** this function. ** -************************************************************************* -** This file contains code used for creating, destroying, and populating -** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior -** to version 2.8.7, all this code was combined into the vdbe.c source file. -** But that file was getting too big so this subroutines were split out. +** Write the number of error seen in *pnErr. Except for some memory +** allocation errors, an error message held in memory obtained from +** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is +** returned. If a memory allocation error occurs, NULL is returned. */ +SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( + Btree *p, /* The btree to be checked */ + int *aRoot, /* An array of root pages numbers for individual trees */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr /* Write number of errors seen to this variable */ +){ + Pgno i; + int nRef; + IntegrityCk sCheck; + BtShared *pBt = p->pBt; + char zErr[100]; + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); + nRef = sqlite3PagerRefcount(pBt->pPager); + sCheck.pBt = pBt; + sCheck.pPager = pBt->pPager; + sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.mxErr = mxErr; + sCheck.nErr = 0; + sCheck.mallocFailed = 0; + *pnErr = 0; + if( sCheck.nPage==0 ){ + sqlite3BtreeLeave(p); + return 0; + } + sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) ); + if( !sCheck.anRef ){ + *pnErr = 1; + sqlite3BtreeLeave(p); + return 0; + } + for(i=0; i<=sCheck.nPage; i++){ sCheck.anRef[i] = 0; } + i = PENDING_BYTE_PAGE(pBt); + if( i<=sCheck.nPage ){ + sCheck.anRef[i] = 1; + } + sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sCheck.errMsg.useMalloc = 2; + /* Check the integrity of the freelist + */ + checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), + get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); -/* -** When debugging the code generator in a symbolic debugger, one can -** set the sqlite3VdbeAddopTrace to 1 and all opcodes will be printed -** as they are added to the instruction stream. -*/ -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3VdbeAddopTrace = 0; + /* Check all the tables. + */ + for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + } #endif + checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); + } + /* Make sure every page in the file is referenced + */ + for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( sCheck.anRef[i]==0 ){ + checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + } +#else + /* If the database supports auto-vacuum, make sure no tables contain + ** references to pointer-map pages. + */ + if( sCheck.anRef[i]==0 && + (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + } + if( sCheck.anRef[i]!=0 && + (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + } +#endif + } -/* -** Create a new virtual database engine. -*/ -SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(sqlite3 *db){ - Vdbe *p; - p = sqlite3DbMallocZero(db, sizeof(Vdbe) ); - if( p==0 ) return 0; - p->db = db; - if( db->pVdbe ){ - db->pVdbe->pPrev = p; + /* Make sure this analysis did not leave any unref() pages. + ** This is an internal consistency check; an integrity check + ** of the integrity check. + */ + if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ + checkAppendMsg(&sCheck, 0, + "Outstanding page count goes from %d to %d during this analysis", + nRef, sqlite3PagerRefcount(pBt->pPager) + ); } - p->pNext = db->pVdbe; - p->pPrev = 0; - db->pVdbe = p; - p->magic = VDBE_MAGIC_INIT; - return p; + + /* Clean up and report errors. + */ + sqlite3BtreeLeave(p); + sqlite3_free(sCheck.anRef); + if( sCheck.mallocFailed ){ + sqlite3StrAccumReset(&sCheck.errMsg); + *pnErr = sCheck.nErr+1; + return 0; + } + *pnErr = sCheck.nErr; + if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + return sqlite3StrAccumFinish(&sCheck.errMsg); } +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* -** Remember the SQL string for a prepared statement. +** Return the full pathname of the underlying database file. +** +** The pager filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. */ -SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ - assert( isPrepareV2==1 || isPrepareV2==0 ); - if( p==0 ) return; -#ifdef SQLITE_OMIT_TRACE - if( !isPrepareV2 ) return; -#endif - assert( p->zSql==0 ); - p->zSql = sqlite3DbStrNDup(p->db, z, n); - p->isPrepareV2 = (u8)isPrepareV2; +SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerFilename(p->pBt->pPager); } /* -** Return the SQL associated with a prepared statement +** Return the pathname of the journal file for this database. The return +** value of this routine is the same regardless of whether the journal file +** has been created or not. +** +** The pager journal filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. */ -SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ - Vdbe *p = (Vdbe *)pStmt; - return (p && p->isPrepareV2) ? p->zSql : 0; +SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerJournalname(p->pBt->pPager); } /* -** Swap all content between two VDBE structures. +** Return non-zero if a transaction is active. */ -SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ - Vdbe tmp, *pTmp; - char *zTmp; - tmp = *pA; - *pA = *pB; - *pB = tmp; - pTmp = pA->pNext; - pA->pNext = pB->pNext; - pB->pNext = pTmp; - pTmp = pA->pPrev; - pA->pPrev = pB->pPrev; - pB->pPrev = pTmp; - zTmp = pA->zSql; - pA->zSql = pB->zSql; - pB->zSql = zTmp; - pB->isPrepareV2 = pA->isPrepareV2; +SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree *p){ + assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); + return (p && (p->inTrans==TRANS_WRITE)); } -#ifdef SQLITE_DEBUG +#ifndef SQLITE_OMIT_WAL /* -** Turn tracing on or off +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -SQLITE_PRIVATE void sqlite3VdbeTrace(Vdbe *p, FILE *trace){ - p->trace = trace; +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager, eMode, pnLog, pnCkpt); + } + sqlite3BtreeLeave(p); + } + return rc; } #endif /* -** Resize the Vdbe.aOp array so that it is at least one op larger than -** it was. -** -** If an out-of-memory error occurs while resizing the array, return -** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain -** unchanged (this is so that any opcodes already allocated can be -** correctly deallocated along with the rest of the Vdbe). +** Return non-zero if a read (or write) transaction is active. */ -static int growOpArray(Vdbe *p){ - VdbeOp *pNew; - int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); - pNew = sqlite3DbRealloc(p->db, p->aOp, nNew*sizeof(Op)); - if( pNew ){ - p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); - p->aOp = pNew; - } - return (pNew ? SQLITE_OK : SQLITE_NOMEM); +SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->inTrans!=TRANS_NONE; } -/* -** Add a new instruction to the list of instructions current in the -** VDBE. Return the address of the new instruction. -** -** Parameters: -** -** p Pointer to the VDBE +SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->nBackup!=0; +} + +/* +** This function returns a pointer to a blob of memory associated with +** a single shared-btree. The memory is used by client code for its own +** purposes (for example, to store a high-level schema associated with +** the shared-btree). The btree layer manages reference counting issues. ** -** op The opcode for this instruction +** The first time this is called on a shared-btree, nBytes bytes of memory +** are allocated, zeroed, and returned to the caller. For each subsequent +** call the nBytes parameter is ignored and a pointer to the same blob +** of memory returned. ** -** p1, p2, p3 Operands +** If the nBytes parameter is 0 and the blob of memory has not yet been +** allocated, a null pointer is returned. If the blob has already been +** allocated, it is returned as normal. ** -** Use the sqlite3VdbeResolveLabel() function to fix an address and -** the sqlite3VdbeChangeP4() function to change the value of the P4 -** operand. +** Just before the shared-btree is closed, the function passed as the +** xFree argument when the memory allocation was made is invoked on the +** blob of allocated memory. The xFree function should not call sqlite3_free() +** on the memory, the btree layer does that. */ -SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ - int i; - VdbeOp *pOp; - - i = p->nOp; - assert( p->magic==VDBE_MAGIC_INIT ); - assert( op>0 && op<0xff ); - if( p->nOpAlloc<=i ){ - if( growOpArray(p) ){ - return 1; - } +SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( !pBt->pSchema && nBytes ){ + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); + pBt->xFreeSchema = xFree; } - p->nOp++; - pOp = &p->aOp[i]; - pOp->opcode = (u8)op; - pOp->p5 = 0; - pOp->p1 = p1; - pOp->p2 = p2; - pOp->p3 = p3; - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; - p->expired = 0; -#ifdef SQLITE_DEBUG - pOp->zComment = 0; - if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); -#endif -#ifdef VDBE_PROFILE - pOp->cycles = 0; - pOp->cnt = 0; -#endif - return i; -} -SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ - return sqlite3VdbeAddOp3(p, op, 0, 0, 0); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ - return sqlite3VdbeAddOp3(p, op, p1, 0, 0); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ - return sqlite3VdbeAddOp3(p, op, p1, p2, 0); + sqlite3BtreeLeave(p); + return pBt->pSchema; } - /* -** Add an opcode that includes the p4 value as a pointer. +** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared +** btree as the argument handle holds an exclusive lock on the +** sqlite_master table. Otherwise SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3VdbeAddOp4( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - const char *zP4, /* The P4 operand */ - int p4type /* P4 operand type */ -){ - int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); - sqlite3VdbeChangeP4(p, addr, zP4, p4type); - return addr; +SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); + assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); + sqlite3BtreeLeave(p); + return rc; } + +#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Add an opcode that includes the p4 value as an integer. +** Obtain a lock on the table whose root page is iTab. The +** lock is a write lock if isWritelock is true or a read lock +** if it is false. */ -SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - int p4 /* The P4 operand as an integer */ -){ - int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); - sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32); - return addr; +SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ + int rc = SQLITE_OK; + assert( p->inTrans!=TRANS_NONE ); + if( p->sharable ){ + u8 lockType = READ_LOCK + isWriteLock; + assert( READ_LOCK+1==WRITE_LOCK ); + assert( isWriteLock==0 || isWriteLock==1 ); + + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, iTab, lockType); + if( rc==SQLITE_OK ){ + rc = setSharedCacheTableLock(p, iTab, lockType); + } + sqlite3BtreeLeave(p); + } + return rc; } +#endif +#ifndef SQLITE_OMIT_INCRBLOB /* -** Create a new symbolic label for an instruction that has yet to be -** coded. The symbolic label is really just a negative number. The -** label can be used as the P2 value of an operation. Later, when -** the label is resolved to a specific address, the VDBE will scan -** through its operation list and change all values of P2 which match -** the label into the resolved address. -** -** The VDBE knows that a P2 value is a label because labels are -** always negative and P2 values are suppose to be non-negative. -** Hence, a negative P2 value is a label that has yet to be resolved. +** Argument pCsr must be a cursor opened for writing on an +** INTKEY table currently pointing at a valid table entry. +** This function modifies the data stored as part of that entry. ** -** Zero is returned if a malloc() fails. +** Only the data content may only be modified, it is not possible to +** change the length of the data stored. If this function is called with +** parameters that attempt to write past the end of the existing data, +** no modifications are made and SQLITE_CORRUPT is returned. */ -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *p){ - int i; - i = p->nLabel++; - assert( p->magic==VDBE_MAGIC_INIT ); - if( i>=p->nLabelAlloc ){ - int n = p->nLabelAlloc*2 + 5; - p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, - n*sizeof(p->aLabel[0])); - p->nLabelAlloc = sqlite3DbMallocSize(p->db, p->aLabel)/sizeof(p->aLabel[0]); +SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ + int rc; + assert( cursorHoldsMutex(pCsr) ); + assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); + assert( pCsr->isIncrblobHandle ); + + rc = restoreCursorPosition(pCsr); + if( rc!=SQLITE_OK ){ + return rc; } - if( p->aLabel ){ - p->aLabel[i] = -1; + assert( pCsr->eState!=CURSOR_REQUIRESEEK ); + if( pCsr->eState!=CURSOR_VALID ){ + return SQLITE_ABORT; } - return -1-i; -} -/* -** Resolve label "x" to be the address of the next instruction to -** be inserted. The parameter "x" must have been obtained from -** a prior call to sqlite3VdbeMakeLabel(). -*/ -SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *p, int x){ - int j = -1-x; - assert( p->magic==VDBE_MAGIC_INIT ); - assert( j>=0 && jnLabel ); - if( p->aLabel ){ - p->aLabel[j] = p->nOp; + /* Check some assumptions: + ** (a) the cursor is open for writing, + ** (b) there is a read/write transaction open, + ** (c) the connection holds a write-lock on the table (if required), + ** (d) there are no conflicting read-locks, and + ** (e) the cursor points at a valid row of an intKey table. + */ + if( !pCsr->wrFlag ){ + return SQLITE_READONLY; } + assert( !pCsr->pBt->readOnly && pCsr->pBt->inTransaction==TRANS_WRITE ); + assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); + assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); + assert( pCsr->apPage[pCsr->iPage]->intKey ); + + return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); } -/* -** Mark the VDBE as one that can only be run one time. +/* +** Set a flag on this cursor to cache the locations of pages from the +** overflow list for the current row. This is used by cursors opened +** for incremental blob IO only. +** +** This function sets a flag only. The actual page location cache +** (stored in BtCursor.aOverflow[]) is allocated and used by function +** accessPayload() (the worker function for sqlite3BtreeData() and +** sqlite3BtreePutData()). */ -SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ - p->runOnlyOnce = 1; +SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + invalidateOverflowCache(pCur); + pCur->isIncrblobHandle = 1; } - -#ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ +#endif /* -** The following type and function are used to iterate through all opcodes -** in a Vdbe main program and each of the sub-programs (triggers) it may -** invoke directly or indirectly. It should be used as follows: -** -** Op *pOp; -** VdbeOpIter sIter; -** -** memset(&sIter, 0, sizeof(sIter)); -** sIter.v = v; // v is of type Vdbe* -** while( (pOp = opIterNext(&sIter)) ){ -** // Do something with pOp -** } -** sqlite3DbFree(v->db, sIter.apSub); -** +** Set both the "read version" (single byte at byte offset 18) and +** "write version" (single byte at byte offset 19) fields in the database +** header to iVersion. */ -typedef struct VdbeOpIter VdbeOpIter; -struct VdbeOpIter { - Vdbe *v; /* Vdbe to iterate through the opcodes of */ - SubProgram **apSub; /* Array of subprograms */ - int nSub; /* Number of entries in apSub */ - int iAddr; /* Address of next instruction to return */ - int iSub; /* 0 = main program, 1 = first sub-program etc. */ -}; -static Op *opIterNext(VdbeOpIter *p){ - Vdbe *v = p->v; - Op *pRet = 0; - Op *aOp; - int nOp; - - if( p->iSub<=p->nSub ){ +SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ + BtShared *pBt = pBtree->pBt; + int rc; /* Return code */ + + assert( pBtree->inTrans==TRANS_NONE ); + assert( iVersion==1 || iVersion==2 ); - if( p->iSub==0 ){ - aOp = v->aOp; - nOp = v->nOp; - }else{ - aOp = p->apSub[p->iSub-1]->aOp; - nOp = p->apSub[p->iSub-1]->nOp; - } - assert( p->iAddrdoNotUseWAL = (u8)(iVersion==1); - pRet = &aOp[p->iAddr]; - p->iAddr++; - if( p->iAddr==nOp ){ - p->iSub++; - p->iAddr = 0; - } - - if( pRet->p4type==P4_SUBPROGRAM ){ - int nByte = (p->nSub+1)*sizeof(SubProgram*); - int j; - for(j=0; jnSub; j++){ - if( p->apSub[j]==pRet->p4.pProgram ) break; - } - if( j==p->nSub ){ - p->apSub = sqlite3DbReallocOrFree(v->db, p->apSub, nByte); - if( !p->apSub ){ - pRet = 0; - }else{ - p->apSub[p->nSub++] = pRet->p4.pProgram; + rc = sqlite3BtreeBeginTrans(pBtree, 0); + if( rc==SQLITE_OK ){ + u8 *aData = pBt->pPage1->aData; + if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ + rc = sqlite3BtreeBeginTrans(pBtree, 2); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + aData[18] = (u8)iVersion; + aData[19] = (u8)iVersion; } } } } - return pRet; + pBt->doNotUseWAL = 0; + return rc; } +/************** End of btree.c ***********************************************/ +/************** Begin file backup.c ******************************************/ /* -** Check if the program stored in the VM associated with pParse may -** throw an ABORT exception (causing the statement, but not entire transaction -** to be rolled back). This condition is true if the main program or any -** sub-programs contains any of the following: +** 2009 January 28 ** -** * OP_Halt with P1=SQLITE_CONSTRAINT and P2=OE_Abort. -** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. -** * OP_Destroy -** * OP_VUpdate -** * OP_VRename -** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Then check that the value of Parse.mayAbort is true if an -** ABORT may be thrown, or false otherwise. Return true if it does -** match, or false otherwise. This function is intended to be used as -** part of an assert statement in the compiler. Similar to: +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** assert( sqlite3VdbeAssertMayAbort(pParse->pVdbe, pParse->mayAbort) ); +************************************************************************* +** This file contains the implementation of the sqlite3_backup_XXX() +** API functions and the related features. */ -SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ - int hasAbort = 0; - Op *pOp; - VdbeOpIter sIter; - memset(&sIter, 0, sizeof(sIter)); - sIter.v = v; - while( (pOp = opIterNext(&sIter))!=0 ){ - int opcode = pOp->opcode; - if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename -#ifndef SQLITE_OMIT_FOREIGN_KEY - || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) +/* Macro to find the minimum of two numeric values. +*/ +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) #endif - || ((opcode==OP_Halt || opcode==OP_HaltIfNull) - && (pOp->p1==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) - ){ - hasAbort = 1; - break; - } - } - sqlite3DbFree(v->db, sIter.apSub); - /* Return true if hasAbort==mayAbort. Or if a malloc failure occured. - ** If malloc failed, then the while() loop above may not have iterated - ** through all opcodes and hasAbort may be set incorrectly. Return - ** true for this case to prevent the assert() in the callers frame - ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort ); -} -#endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ +/* +** Structure allocated for each backup operation. +*/ +struct sqlite3_backup { + sqlite3* pDestDb; /* Destination database handle */ + Btree *pDest; /* Destination b-tree file */ + u32 iDestSchema; /* Original schema cookie in destination */ + int bDestLocked; /* True once a write-transaction is open on pDest */ + + Pgno iNext; /* Page number of the next source page to copy */ + sqlite3* pSrcDb; /* Source database handle */ + Btree *pSrc; /* Source b-tree file */ + + int rc; /* Backup process error code */ + + /* These two variables are set by every call to backup_step(). They are + ** read by calls to backup_remaining() and backup_pagecount(). + */ + Pgno nRemaining; /* Number of pages left to copy */ + Pgno nPagecount; /* Total number of pages to copy */ + + int isAttached; /* True once backup has been registered with pager */ + sqlite3_backup *pNext; /* Next backup associated with source pager */ +}; /* -** Loop through the program looking for P2 values that are negative -** on jump instructions. Each such value is a label. Resolve the -** label by setting the P2 value to its correct non-zero value. +** THREAD SAFETY NOTES: ** -** This routine is called once after all opcodes have been inserted. +** Once it has been created using backup_init(), a single sqlite3_backup +** structure may be accessed via two groups of thread-safe entry points: ** -** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument -** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by -** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** * Via the sqlite3_backup_XXX() API function backup_step() and +** backup_finish(). Both these functions obtain the source database +** handle mutex and the mutex associated with the source BtShared +** structure, in that order. ** -** The Op.opflags field is set on all opcodes. +** * Via the BackupUpdate() and BackupRestart() functions, which are +** invoked by the pager layer to report various state changes in +** the page cache associated with the source database. The mutex +** associated with the source database BtShared structure will always +** be held when either of these functions are invoked. +** +** The other sqlite3_backup_XXX() API functions, backup_remaining() and +** backup_pagecount() are not thread-safe functions. If they are called +** while some other thread is calling backup_step() or backup_finish(), +** the values returned may be invalid. There is no way for a call to +** BackupUpdate() or BackupRestart() to interfere with backup_remaining() +** or backup_pagecount(). +** +** Depending on the SQLite configuration, the database handles and/or +** the Btree objects may have their own mutexes that require locking. +** Non-sharable Btrees (in-memory databases for example), do not have +** associated mutexes. */ -static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ - int i; - int nMaxArgs = *pMaxFuncArgs; - Op *pOp; - int *aLabel = p->aLabel; - p->readOnly = 1; - for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ - u8 opcode = pOp->opcode; - pOp->opflags = sqlite3OpcodeProperty[opcode]; - if( opcode==OP_Function || opcode==OP_AggStep ){ - if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; - }else if( opcode==OP_Transaction && pOp->p2!=0 ){ - p->readOnly = 0; -#ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( opcode==OP_VUpdate ){ - if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; - }else if( opcode==OP_VFilter ){ - int n; - assert( p->nOp - i >= 3 ); - assert( pOp[-1].opcode==OP_Integer ); - n = pOp[-1].p1; - if( n>nMaxArgs ) nMaxArgs = n; -#endif - } +/* +** Return a pointer corresponding to database zDb (i.e. "main", "temp") +** in connection handle pDb. If such a database cannot be found, return +** a NULL pointer and write an error message to pErrorDb. +** +** If the "temp" database is requested, it may need to be opened by this +** function. If an error occurs while doing so, return 0 and write an +** error message to pErrorDb. +*/ +static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ + int i = sqlite3FindDbName(pDb, zDb); - if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ - assert( -1-pOp->p2nLabel ); - pOp->p2 = aLabel[-1-pOp->p2]; + if( i==1 ){ + Parse *pParse; + int rc = 0; + pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); + if( pParse==0 ){ + sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory"); + rc = SQLITE_NOMEM; + }else{ + pParse->db = pDb; + if( sqlite3OpenTempDatabase(pParse) ){ + sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); + rc = SQLITE_ERROR; + } + sqlite3DbFree(pErrorDb, pParse->zErrMsg); + sqlite3StackFree(pErrorDb, pParse); + } + if( rc ){ + return 0; } } - sqlite3DbFree(p->db, p->aLabel); - p->aLabel = 0; - *pMaxFuncArgs = nMaxArgs; + if( i<0 ){ + sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + return 0; + } + + return pDb->aDb[i].pBt; } /* -** Return the address of the next instruction to be inserted. +** Attempt to set the page size of the destination to match the page size +** of the source. */ -SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ - assert( p->magic==VDBE_MAGIC_INIT ); - return p->nOp; +static int setDestPgsz(sqlite3_backup *p){ + int rc; + rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0); + return rc; } /* -** This function returns a pointer to the array of opcodes associated with -** the Vdbe passed as the first argument. It is the callers responsibility -** to arrange for the returned array to be eventually freed using the -** vdbeFreeOpArray() function. +** Create an sqlite3_backup process to copy the contents of zSrcDb from +** connection handle pSrcDb to zDestDb in pDestDb. If successful, return +** a pointer to the new sqlite3_backup object. ** -** Before returning, *pnOp is set to the number of entries in the returned -** array. Also, *pnMaxArg is set to the larger of its current value and -** the number of entries in the Vdbe.apArg[] array required to execute the -** returned program. +** If an error occurs, NULL is returned and an error code and error message +** stored in database handle pDestDb. */ -SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ - VdbeOp *aOp = p->aOp; - assert( aOp && !p->db->mallocFailed ); - - /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ - assert( p->aMutex.nMutex==0 ); +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3* pDestDb, /* Database to write to */ + const char *zDestDb, /* Name of database within pDestDb */ + sqlite3* pSrcDb, /* Database connection to read from */ + const char *zSrcDb /* Name of database within pSrcDb */ +){ + sqlite3_backup *p; /* Value to return */ - resolveP2Values(p, pnMaxArg); - *pnOp = p->nOp; - p->aOp = 0; - return aOp; -} + /* Lock the source database handle. The destination database + ** handle is not locked in this routine, but it is locked in + ** sqlite3_backup_step(). The user is required to ensure that no + ** other thread accesses the destination handle for the duration + ** of the backup operation. Any attempt to use the destination + ** database connection while a backup is in progress may cause + ** a malfunction or a deadlock. + */ + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3_mutex_enter(pDestDb->mutex); -/* -** Add a whole list of operations to the operation stack. Return the -** address of the first operation added. -*/ -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){ - int addr; - assert( p->magic==VDBE_MAGIC_INIT ); - if( p->nOp + nOp > p->nOpAlloc && growOpArray(p) ){ - return 0; + if( pSrcDb==pDestDb ){ + sqlite3Error( + pDestDb, SQLITE_ERROR, "source and destination must be distinct" + ); + p = 0; + }else { + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); + if( !p ){ + sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + } } - addr = p->nOp; - if( ALWAYS(nOp>0) ){ - int i; - VdbeOpList const *pIn = aOp; - for(i=0; ip2; - VdbeOp *pOut = &p->aOp[i+addr]; - pOut->opcode = pIn->opcode; - pOut->p1 = pIn->p1; - if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){ - pOut->p2 = addr + ADDR(p2); - }else{ - pOut->p2 = p2; - } - pOut->p3 = pIn->p3; - pOut->p4type = P4_NOTUSED; - pOut->p4.p = 0; - pOut->p5 = 0; -#ifdef SQLITE_DEBUG - pOut->zComment = 0; - if( sqlite3VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); - } -#endif + + /* If the allocation succeeded, populate the new object. */ + if( p ){ + memset(p, 0, sizeof(sqlite3_backup)); + p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); + p->pDest = findBtree(pDestDb, pDestDb, zDestDb); + p->pDestDb = pDestDb; + p->pSrcDb = pSrcDb; + p->iNext = 1; + p->isAttached = 0; + + if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + /* One (or both) of the named databases did not exist or an OOM + ** error was hit. The error has already been written into the + ** pDestDb handle. All that is left to do here is free the + ** sqlite3_backup structure. + */ + sqlite3_free(p); + p = 0; } - p->nOp += nOp; } - return addr; + if( p ){ + p->pSrc->nBackup++; + } + + sqlite3_mutex_leave(pDestDb->mutex); + sqlite3_mutex_leave(pSrcDb->mutex); + return p; } /* -** Change the value of the P1 operand for a specific instruction. -** This routine is useful when a large program is loaded from a -** static array using sqlite3VdbeAddOpList but we want to make a -** few minor changes to the program. +** Argument rc is an SQLite error code. Return true if this error is +** considered fatal if encountered during a backup operation. All errors +** are considered fatal except for SQLITE_BUSY and SQLITE_LOCKED. */ -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ - assert( p!=0 ); - assert( addr>=0 ); - if( p->nOp>addr ){ - p->aOp[addr].p1 = val; - } +static int isFatalError(int rc){ + return (rc!=SQLITE_OK && rc!=SQLITE_BUSY && ALWAYS(rc!=SQLITE_LOCKED)); } /* -** Change the value of the P2 operand for a specific instruction. -** This routine is useful for setting a jump destination. +** Parameter zSrcData points to a buffer containing the data for +** page iSrcPg from the source database. Copy this data into the +** destination database. */ -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ - assert( p!=0 ); - assert( addr>=0 ); - if( p->nOp>addr ){ - p->aOp[addr].p2 = val; +static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); + const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; +#ifdef SQLITE_HAS_CODEC + int nSrcReserve = sqlite3BtreeGetReserve(p->pSrc); + int nDestReserve = sqlite3BtreeGetReserve(p->pDest); +#endif + + int rc = SQLITE_OK; + i64 iOff; + + assert( p->bDestLocked ); + assert( !isFatalError(p->rc) ); + assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); + assert( zSrcData ); + + /* Catch the case where the destination is an in-memory database and the + ** page sizes of the source and destination differ. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ + rc = SQLITE_READONLY; } -} -/* -** Change the value of the P3 operand for a specific instruction. -*/ -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ - assert( p!=0 ); - assert( addr>=0 ); - if( p->nOp>addr ){ - p->aOp[addr].p3 = val; +#ifdef SQLITE_HAS_CODEC + /* Backup is not possible if the page size of the destination is changing + ** and a codec is in use. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ + rc = SQLITE_READONLY; + } + + /* Backup is not possible if the number of bytes of reserve space differ + ** between source and destination. If there is a difference, try to + ** fix the destination to agree with the source. If that is not possible, + ** then the backup cannot proceed. + */ + if( nSrcReserve!=nDestReserve ){ + u32 newPgsz = nSrcPgsz; + rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); + if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; + } +#endif + + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. + */ + for(iOff=iEnd-(i64)nSrcPgsz; rc==SQLITE_OK && iOffpDest->pBt) ) continue; + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg)) + && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) + ){ + const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; + u8 *zDestData = sqlite3PagerGetData(pDestPg); + u8 *zOut = &zDestData[iOff%nDestPgsz]; + + /* Copy the data from the source page into the destination page. + ** Then clear the Btree layer MemPage.isInit flag. Both this module + ** and the pager code use this trick (clearing the first byte + ** of the page 'extra' space to invalidate the Btree layers + ** cached parse of the page). MemPage.isInit is marked + ** "MUST BE FIRST" for this purpose. + */ + memcpy(zOut, zIn, nCopy); + ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + } + sqlite3PagerUnref(pDestPg); } + + return rc; } /* -** Change the value of the P5 operand for the most recently -** added operation. +** If pFile is currently larger than iSize bytes, then truncate it to +** exactly iSize bytes. If pFile is not larger than iSize bytes, then +** this function is a no-op. +** +** Return SQLITE_OK if everything is successful, or an SQLite error +** code if an error occurs. */ -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ - assert( p!=0 ); - if( p->aOp ){ - assert( p->nOp>0 ); - p->aOp[p->nOp-1].p5 = val; +static int backupTruncateFile(sqlite3_file *pFile, i64 iSize){ + i64 iCurrent; + int rc = sqlite3OsFileSize(pFile, &iCurrent); + if( rc==SQLITE_OK && iCurrent>iSize ){ + rc = sqlite3OsTruncate(pFile, iSize); } + return rc; } /* -** Change the P2 operand of instruction addr so that it points to -** the address of the next instruction to be coded. +** Register this backup object with the associated source pager for +** callbacks when pages are changed or the cache invalidated. */ -SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - sqlite3VdbeChangeP2(p, addr, p->nOp); +static void attachBackupObject(sqlite3_backup *p){ + sqlite3_backup **pp; + assert( sqlite3BtreeHoldsMutex(p->pSrc) ); + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + p->pNext = *pp; + *pp = p; + p->isAttached = 1; } - /* -** If the input FuncDef structure is ephemeral, then free it. If -** the FuncDef is not ephermal, then do nothing. +** Copy nPage pages from the source b-tree to the destination. */ -static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ - if( ALWAYS(pDef) && (pDef->flags & SQLITE_FUNC_EPHEM)!=0 ){ - sqlite3DbFree(db, pDef); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + int rc; + int destMode; /* Destination journal mode */ + int pgszSrc = 0; /* Source page size */ + int pgszDest = 0; /* Destination page size */ + + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); } -} -static void vdbeFreeOpArray(sqlite3 *, Op *, int); + rc = p->rc; + if( !isFatalError(rc) ){ + Pager * const pSrcPager = sqlite3BtreePager(p->pSrc); /* Source pager */ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); /* Dest pager */ + int ii; /* Iterator variable */ + int nSrcPage = -1; /* Size of source db in pages */ + int bCloseTrans = 0; /* True if src db requires unlocking */ -/* -** Delete a P4 value if necessary. -*/ -static void freeP4(sqlite3 *db, int p4type, void *p4){ - if( p4 ){ - assert( db ); - switch( p4type ){ - case P4_REAL: - case P4_INT64: - case P4_DYNAMIC: - case P4_KEYINFO: - case P4_INTARRAY: - case P4_KEYINFO_HANDOFF: { - sqlite3DbFree(db, p4); - break; + /* If the source pager is currently in a write-transaction, return + ** SQLITE_BUSY immediately. + */ + if( p->pDestDb && p->pSrc->pBt->inTransaction==TRANS_WRITE ){ + rc = SQLITE_BUSY; + }else{ + rc = SQLITE_OK; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + } + + /* If there is no open read-transaction on the source database, open + ** one now. If a transaction is opened here, then it will be closed + ** before this function exits. + */ + if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ + rc = sqlite3BtreeBeginTrans(p->pSrc, 0); + bCloseTrans = 1; + } + + /* Do not allow backup if the destination database is in WAL mode + ** and the page sizes are different between source and destination */ + pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); + pgszDest = sqlite3BtreeGetPageSize(p->pDest); + destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); + if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + rc = SQLITE_READONLY; + } + + /* Now that there is a read-lock on the source database, query the + ** source pager for the number of pages in the database. + */ + nSrcPage = (int)sqlite3BtreeLastPage(p->pSrc); + assert( nSrcPage>=0 ); + for(ii=0; (nPage<0 || iiiNext<=(Pgno)nSrcPage && !rc; ii++){ + const Pgno iSrcPg = p->iNext; /* Source page number */ + if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ + DbPage *pSrcPg; /* Source page object */ + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + if( rc==SQLITE_OK ){ + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg)); + sqlite3PagerUnref(pSrcPg); + } } - case P4_MPRINTF: { - if( db->pnBytesFreed==0 ) sqlite3_free(p4); - break; + p->iNext++; + } + if( rc==SQLITE_OK ){ + p->nPagecount = nSrcPage; + p->nRemaining = nSrcPage+1-p->iNext; + if( p->iNext>(Pgno)nSrcPage ){ + rc = SQLITE_DONE; + }else if( !p->isAttached ){ + attachBackupObject(p); } - case P4_VDBEFUNC: { - VdbeFunc *pVdbeFunc = (VdbeFunc *)p4; - freeEphemeralFunction(db, pVdbeFunc->pFunc); - if( db->pnBytesFreed==0 ) sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); - sqlite3DbFree(db, pVdbeFunc); - break; + } + + /* Update the schema version field in the destination database. This + ** is to make sure that the schema-version really does change in + ** the case where the source and destination databases have the + ** same schema version. + */ + if( rc==SQLITE_DONE + && (rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1))==SQLITE_OK + ){ + int nDestTruncate; + + if( p->pDestDb ){ + sqlite3ResetInternalSchema(p->pDestDb, -1); } - case P4_FUNCDEF: { - freeEphemeralFunction(db, (FuncDef*)p4); - break; + + /* Set nDestTruncate to the final number of pages in the destination + ** database. The complication here is that the destination page + ** size may be different to the source page size. + ** + ** If the source page size is smaller than the destination page size, + ** round up. In this case the call to sqlite3OsTruncate() below will + ** fix the size of the file. However it is important to call + ** sqlite3PagerTruncateImage() here so that any pages in the + ** destination file that lie beyond the nDestTruncate page mark are + ** journalled by PagerCommitPhaseOne() before they are destroyed + ** by the file truncation. + */ + assert( pgszSrc==sqlite3BtreeGetPageSize(p->pSrc) ); + assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); + if( pgszSrcpDest->pBt) ){ + nDestTruncate--; + } + }else{ + nDestTruncate = nSrcPage * (pgszSrc/pgszDest); } - case P4_MEM: { - if( db->pnBytesFreed==0 ){ - sqlite3ValueFree((sqlite3_value*)p4); - }else{ - Mem *p = (Mem*)p4; - sqlite3DbFree(db, p->zMalloc); - sqlite3DbFree(db, p); + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); + + if( pgszSrc= iSize || ( + nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) + && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest + )); + + /* This call ensures that all data required to recreate the original + ** database has been stored in the journal for pDestPager and the + ** journal synced to disk. So at this point we may safely modify + ** the database file in any way, knowing that if a power failure + ** occurs, the original database will be reconstructed from the + ** journal file. */ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + + /* Write the extra pages and truncate the database file as required. */ + iEnd = MIN(PENDING_BYTE + pgszDest, iSize); + for( + iOff=PENDING_BYTE+pgszSrc; + rc==SQLITE_OK && iOffpnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); - break; + + /* Finish committing the transaction to the destination database. */ + if( SQLITE_OK==rc + && SQLITE_OK==(rc = sqlite3BtreeCommitPhaseTwo(p->pDest, 0)) + ){ + rc = SQLITE_DONE; } } + + /* If bCloseTrans is true, then this function opened a read transaction + ** on the source database. Close the read transaction here. There is + ** no need to check the return values of the btree methods here, as + ** "committing" a read-only transaction cannot fail. + */ + if( bCloseTrans ){ + TESTONLY( int rc2 ); + TESTONLY( rc2 = ) sqlite3BtreeCommitPhaseOne(p->pSrc, 0); + TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc, 0); + assert( rc2==SQLITE_OK ); + } + + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; + } + p->rc = rc; + } + if( p->pDestDb ){ + sqlite3_mutex_leave(p->pDestDb->mutex); } + sqlite3BtreeLeave(p->pSrc); + sqlite3_mutex_leave(p->pSrcDb->mutex); + return rc; } /* -** Free the space allocated for aOp and any p4 values allocated for the -** opcodes contained within. If aOp is not NULL it is assumed to contain -** nOp entries. +** Release all resources associated with an sqlite3_backup* handle. */ -static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ - if( aOp ){ - Op *pOp; - for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - freeP4(db, pOp->p4type, pOp->p4.p); -#ifdef SQLITE_DEBUG - sqlite3DbFree(db, pOp->zComment); -#endif +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ + sqlite3_backup **pp; /* Ptr to head of pagers backup list */ + sqlite3_mutex *mutex; /* Mutex to protect source database */ + int rc; /* Value to return */ + + /* Enter the mutexes */ + if( p==0 ) return SQLITE_OK; + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + mutex = p->pSrcDb->mutex; + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + /* Detach this backup from the source pager. */ + if( p->pDestDb ){ + p->pSrc->nBackup--; + } + if( p->isAttached ){ + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + while( *pp!=p ){ + pp = &(*pp)->pNext; } + *pp = p->pNext; } - sqlite3DbFree(db, aOp); + + /* If a transaction is still open on the Btree, roll it back. */ + sqlite3BtreeRollback(p->pDest); + + /* Set the error code of the destination database handle. */ + rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; + sqlite3Error(p->pDestDb, rc, 0); + + /* Exit the mutexes and free the backup context structure. */ + if( p->pDestDb ){ + sqlite3_mutex_leave(p->pDestDb->mutex); + } + sqlite3BtreeLeave(p->pSrc); + if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + sqlite3_free(p); + } + sqlite3_mutex_leave(mutex); + return rc; } /* -** Link the SubProgram object passed as the second argument into the linked -** list at Vdbe.pSubProgram. This list is used to delete all sub-program -** objects when the VM is no longer required. +** Return the number of pages still to be backed up as of the most recent +** call to sqlite3_backup_step(). */ -SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ - p->pNext = pVdbe->pProgram; - pVdbe->pProgram = p; +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ + return p->nRemaining; } /* -** Change N opcodes starting at addr to No-ops. +** Return the total number of pages in the source database as of the most +** recent call to sqlite3_backup_step(). */ -SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr, int N){ - if( p->aOp ){ - VdbeOp *pOp = &p->aOp[addr]; - sqlite3 *db = p->db; - while( N-- ){ - freeP4(db, pOp->p4type, pOp->p4.p); - memset(pOp, 0, sizeof(pOp[0])); - pOp->opcode = OP_Noop; - pOp++; - } - } +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ + return p->nPagecount; } /* -** Change the value of the P4 operand for a specific instruction. -** This routine is useful when a large program is loaded from a -** static array using sqlite3VdbeAddOpList but we want to make a -** few minor changes to the program. -** -** If n>=0 then the P4 operand is dynamic, meaning that a copy of -** the string is made into memory obtained from sqlite3_malloc(). -** A value of n==0 means copy bytes of zP4 up to and including the -** first null byte. If n>0 then copy n+1 bytes of zP4. -** -** If n==P4_KEYINFO it means that zP4 is a pointer to a KeyInfo structure. -** A copy is made of the KeyInfo structure into memory obtained from -** sqlite3_malloc, to be freed when the Vdbe is finalized. -** n==P4_KEYINFO_HANDOFF indicates that zP4 points to a KeyInfo structure -** stored in memory that the caller has obtained from sqlite3_malloc. The -** caller should not free the allocation, it will be freed when the Vdbe is -** finalized. -** -** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points -** to a string or structure that is guaranteed to exist for the lifetime of -** the Vdbe. In these cases we can just copy the pointer. +** This function is called after the contents of page iPage of the +** source database have been modified. If page iPage has already been +** copied into the destination database, then the data written to the +** destination is now invalidated. The destination copy of iPage needs +** to be updated with the new data before the backup operation is +** complete. ** -** If addr<0 then change P4 on the most recently inserted instruction. +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. */ -SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ - Op *pOp; - sqlite3 *db; - assert( p!=0 ); - db = p->db; - assert( p->magic==VDBE_MAGIC_INIT ); - if( p->aOp==0 || db->mallocFailed ){ - if ( n!=P4_KEYINFO && n!=P4_VTAB ) { - freeP4(db, n, (void*)*(char**)&zP4); - } - return; - } - assert( p->nOp>0 ); - assert( addrnOp ); - if( addr<0 ){ - addr = p->nOp - 1; - } - pOp = &p->aOp[addr]; - freeP4(db, pOp->p4type, pOp->p4.p); - pOp->p4.p = 0; - if( n==P4_INT32 ){ - /* Note: this cast is safe, because the origin data point was an int - ** that was cast to a (const char *). */ - pOp->p4.i = SQLITE_PTR_TO_INT(zP4); - pOp->p4type = P4_INT32; - }else if( zP4==0 ){ - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; - }else if( n==P4_KEYINFO ){ - KeyInfo *pKeyInfo; - int nField, nByte; - - nField = ((KeyInfo*)zP4)->nField; - nByte = sizeof(*pKeyInfo) + (nField-1)*sizeof(pKeyInfo->aColl[0]) + nField; - pKeyInfo = sqlite3DbMallocRaw(0, nByte); - pOp->p4.pKeyInfo = pKeyInfo; - if( pKeyInfo ){ - u8 *aSortOrder; - memcpy((char*)pKeyInfo, zP4, nByte - nField); - aSortOrder = pKeyInfo->aSortOrder; - if( aSortOrder ){ - pKeyInfo->aSortOrder = (unsigned char*)&pKeyInfo->aColl[nField]; - memcpy(pKeyInfo->aSortOrder, aSortOrder, nField); +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + if( !isFatalError(p->rc) && iPageiNext ){ + /* The backup process p has already copied page iPage. But now it + ** has been modified by a transaction on the source pager. Copy + ** the new data into the backup. + */ + int rc; + assert( p->pDestDb ); + sqlite3_mutex_enter(p->pDestDb->mutex); + rc = backupOnePage(p, iPage, aData); + sqlite3_mutex_leave(p->pDestDb->mutex); + assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); + if( rc!=SQLITE_OK ){ + p->rc = rc; } - pOp->p4type = P4_KEYINFO; - }else{ - p->db->mallocFailed = 1; - pOp->p4type = P4_NOTUSED; } - }else if( n==P4_KEYINFO_HANDOFF ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_KEYINFO; - }else if( n==P4_VTAB ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_VTAB; - sqlite3VtabLock((VTable *)zP4); - assert( ((VTable *)zP4)->db==p->db ); - }else if( n<0 ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = (signed char)n; - }else{ - if( n==0 ) n = sqlite3Strlen30(zP4); - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); - pOp->p4type = P4_DYNAMIC; } } -#ifndef NDEBUG /* -** Change the comment on the the most recently coded instruction. Or -** insert a No-op and add the comment to that new instruction. This -** makes the code easier to read during debugging. None of this happens -** in a production build. +** Restart the backup process. This is called when the pager layer +** detects that the database has been modified by an external database +** connection. In this case there is no way of knowing which of the +** pages that have been copied into the destination database are still +** valid and which are not, so the entire process needs to be restarted. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. */ -SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ - va_list ap; - if( !p ) return; - assert( p->nOp>0 || p->aOp==0 ); - assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); - if( p->nOp ){ - char **pz = &p->aOp[p->nOp-1].zComment; - va_start(ap, zFormat); - sqlite3DbFree(p->db, *pz); - *pz = sqlite3VMPrintf(p->db, zFormat, ap); - va_end(ap); +SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *pBackup){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + p->iNext = 1; } } -SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ - va_list ap; - if( !p ) return; - sqlite3VdbeAddOp0(p, OP_Noop); - assert( p->nOp>0 || p->aOp==0 ); - assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); - if( p->nOp ){ - char **pz = &p->aOp[p->nOp-1].zComment; - va_start(ap, zFormat); - sqlite3DbFree(p->db, *pz); - *pz = sqlite3VMPrintf(p->db, zFormat, ap); - va_end(ap); + +#ifndef SQLITE_OMIT_VACUUM +/* +** Copy the complete content of pBtFrom into pBtTo. A transaction +** must be active for both files. +** +** The size of file pTo may be reduced by this operation. If anything +** goes wrong, the transaction on pTo is rolled back. If successful, the +** transaction is committed before returning. +*/ +SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + int rc; + sqlite3_backup b; + sqlite3BtreeEnter(pTo); + sqlite3BtreeEnter(pFrom); + + /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set + ** to 0. This is used by the implementations of sqlite3_backup_step() + ** and sqlite3_backup_finish() to detect that they are being called + ** from this function, not directly by the user. + */ + memset(&b, 0, sizeof(b)); + b.pSrcDb = pFrom->db; + b.pSrc = pFrom; + b.pDest = pTo; + b.iNext = 1; + + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes + ** within a single call (unless an error occurs). The assert() statement + ** checks this assumption - (p->rc) should be set to either SQLITE_DONE + ** or an error code. + */ + sqlite3_backup_step(&b, 0x7FFFFFFF); + assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); + if( rc==SQLITE_OK ){ + pTo->pBt->pageSizeFixed = 0; } + + sqlite3BtreeLeave(pFrom); + sqlite3BtreeLeave(pTo); + return rc; } -#endif /* NDEBUG */ +#endif /* SQLITE_OMIT_VACUUM */ +/************** End of backup.c **********************************************/ +/************** Begin file vdbemem.c *****************************************/ /* -** Return the opcode for a given address. If the address is -1, then -** return the most recently inserted opcode. +** 2004 May 26 ** -** If a memory allocation error has occurred prior to the calling of this -** routine, then a pointer to a dummy VdbeOp will be returned. That opcode -** is readable but not writable, though it is cast to a writable value. -** The return of a dummy opcode allows the call to continue functioning -** after a OOM fault without having to check to see if the return from -** this routine is a valid pointer. But because the dummy.opcode is 0, -** dummy will never be written to. This is verified by code inspection and -** by running with Valgrind. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** About the #ifdef SQLITE_OMIT_TRACE: Normally, this routine is never called -** unless p->nOp>0. This is because in the absense of SQLITE_OMIT_TRACE, -** an OP_Trace instruction is always inserted by sqlite3VdbeGet() as soon as -** a new VDBE is created. So we are free to set addr to p->nOp-1 without -** having to double-check to make sure that the result is non-negative. But -** if SQLITE_OMIT_TRACE is defined, the OP_Trace is omitted and we do need to -** check the value of p->nOp-1 before continuing. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to manipulate "Mem" structure. A "Mem" +** stores a single value in the VDBE. Mem is an opaque structure visible +** only within the VDBE. Interface routines refer to a Mem using the +** name sqlite_value */ -SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ - /* C89 specifies that the constant "dummy" will be initialized to all - ** zeros, which is correct. MSVC generates a warning, nevertheless. */ - static const VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ - assert( p->magic==VDBE_MAGIC_INIT ); - if( addr<0 ){ -#ifdef SQLITE_OMIT_TRACE - if( p->nOp==0 ) return (VdbeOp*)&dummy; + +/* +** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*) +** P if required. +*/ +#define expandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) + +/* +** If pMem is an object with a valid string representation, this routine +** ensures the internal encoding for the string representation is +** 'desiredEnc', one of SQLITE_UTF8, SQLITE_UTF16LE or SQLITE_UTF16BE. +** +** If pMem is not a string object, or the encoding of the string +** representation is already stored using the requested encoding, then this +** routine is a no-op. +** +** SQLITE_OK is returned if the conversion is successful (or not required). +** SQLITE_NOMEM may be returned if a malloc() fails during conversion +** between formats. +*/ +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ + int rc; + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE + || desiredEnc==SQLITE_UTF16BE ); + if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ + return SQLITE_OK; + } + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); +#ifdef SQLITE_OMIT_UTF16 + return SQLITE_ERROR; +#else + + /* MemTranslate() may return SQLITE_OK or SQLITE_NOMEM. If NOMEM is returned, + ** then the encoding of the value may not have changed. + */ + rc = sqlite3VdbeMemTranslate(pMem, (u8)desiredEnc); + assert(rc==SQLITE_OK || rc==SQLITE_NOMEM); + assert(rc==SQLITE_OK || pMem->enc!=desiredEnc); + assert(rc==SQLITE_NOMEM || pMem->enc==desiredEnc); + return rc; #endif - addr = p->nOp - 1; +} + +/* +** Make sure pMem->z points to a writable allocation of at least +** n bytes. +** +** If the memory cell currently contains string or blob data +** and the third argument passed to this function is true, the +** current content of the cell is preserved. Otherwise, it may +** be discarded. +** +** This function sets the MEM_Dyn flag and clears any xDel callback. +** It also clears MEM_Ephem and MEM_Static. If the preserve flag is +** not set, Mem.n is zeroed. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve){ + assert( 1 >= + ((pMem->zMalloc && pMem->zMalloc==pMem->z) ? 1 : 0) + + (((pMem->flags&MEM_Dyn)&&pMem->xDel) ? 1 : 0) + + ((pMem->flags&MEM_Ephem) ? 1 : 0) + + ((pMem->flags&MEM_Static) ? 1 : 0) + ); + assert( (pMem->flags&MEM_RowSet)==0 ); + + if( n<32 ) n = 32; + if( sqlite3DbMallocSize(pMem->db, pMem->zMalloc)z==pMem->zMalloc ){ + pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + preserve = 0; + }else{ + sqlite3DbFree(pMem->db, pMem->zMalloc); + pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); + } } - assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); - if( p->db->mallocFailed ){ - return (VdbeOp*)&dummy; + + if( pMem->z && preserve && pMem->zMalloc && pMem->z!=pMem->zMalloc ){ + memcpy(pMem->zMalloc, pMem->z, pMem->n); + } + if( pMem->flags&MEM_Dyn && pMem->xDel ){ + pMem->xDel((void *)(pMem->z)); + } + + pMem->z = pMem->zMalloc; + if( pMem->z==0 ){ + pMem->flags = MEM_Null; }else{ - return &p->aOp[addr]; + pMem->flags &= ~(MEM_Ephem|MEM_Static); } + pMem->xDel = 0; + return (pMem->z ? SQLITE_OK : SQLITE_NOMEM); } -#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ - || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) /* -** Compute a string that describes the P4 parameter for an opcode. -** Use zTemp for any required temporary buffer space. +** Make the given Mem object MEM_Dyn. In other words, make it so +** that any TEXT or BLOB content is stored in memory obtained from +** malloc(). In this way, we know that the memory is safe to be +** overwritten or altered. +** +** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. */ -static char *displayP4(Op *pOp, char *zTemp, int nTemp){ - char *zP4 = zTemp; - assert( nTemp>=20 ); - switch( pOp->p4type ){ - case P4_KEYINFO_STATIC: - case P4_KEYINFO: { - int i, j; - KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; - sqlite3_snprintf(nTemp, zTemp, "keyinfo(%d", pKeyInfo->nField); - i = sqlite3Strlen30(zTemp); - for(j=0; jnField; j++){ - CollSeq *pColl = pKeyInfo->aColl[j]; - if( pColl ){ - int n = sqlite3Strlen30(pColl->zName); - if( i+n>nTemp-6 ){ - memcpy(&zTemp[i],",...",4); - break; - } - zTemp[i++] = ','; - if( pKeyInfo->aSortOrder && pKeyInfo->aSortOrder[j] ){ - zTemp[i++] = '-'; - } - memcpy(&zTemp[i], pColl->zName,n+1); - i += n; - }else if( i+4p4.pColl; - sqlite3_snprintf(nTemp, zTemp, "collseq(%.20s)", pColl->zName); - break; - } - case P4_FUNCDEF: { - FuncDef *pDef = pOp->p4.pFunc; - sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); - break; - } - case P4_INT64: { - sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); - break; - } - case P4_INT32: { - sqlite3_snprintf(nTemp, zTemp, "%d", pOp->p4.i); - break; - } - case P4_REAL: { - sqlite3_snprintf(nTemp, zTemp, "%.16g", *pOp->p4.pReal); - break; - } - case P4_MEM: { - Mem *pMem = pOp->p4.pMem; - assert( (pMem->flags & MEM_Null)==0 ); - if( pMem->flags & MEM_Str ){ - zP4 = pMem->z; - }else if( pMem->flags & MEM_Int ){ - sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); - }else if( pMem->flags & MEM_Real ){ - sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); - }else{ - assert( pMem->flags & MEM_Blob ); - zP4 = "(blob)"; - } - break; - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - case P4_VTAB: { - sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; - sqlite3_snprintf(nTemp, zTemp, "vtab:%p:%p", pVtab, pVtab->pModule); - break; +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ + int f; + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( (pMem->flags&MEM_RowSet)==0 ); + expandBlob(pMem); + f = pMem->flags; + if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ + return SQLITE_NOMEM; } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->flags |= MEM_Term; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; #endif - case P4_INTARRAY: { - sqlite3_snprintf(nTemp, zTemp, "intarray"); - break; - } - case P4_SUBPROGRAM: { - sqlite3_snprintf(nTemp, zTemp, "program"); - break; - } - default: { - zP4 = pOp->p4.z; - if( zP4==0 ){ - zP4 = zTemp; - zTemp[0] = 0; - } - } } - assert( zP4!=0 ); - return zP4; + + return SQLITE_OK; } -#endif /* -** Declare to the Vdbe that the BTree object at db->aDb[i] is used. -** -** The prepared statement has to know in advance which Btree objects -** will be used so that it can acquire mutexes on them all in sorted -** order (via sqlite3VdbeMutexArrayEnter(). Mutexes are acquired -** in order (and released in reverse order) to avoid deadlocks. +** If the given Mem* has a zero-filled tail, turn it into an ordinary +** blob stored in dynamically allocated space. */ -SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ - int mask; - assert( i>=0 && idb->nDb && ibtreeMask)*8 ); - mask = ((u32)1)<btreeMask & mask)==0 ){ - p->btreeMask |= mask; - sqlite3BtreeMutexArrayInsert(&p->aMutex, p->db->aDb[i].pBt); +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ + if( pMem->flags & MEM_Zero ){ + int nByte; + assert( pMem->flags&MEM_Blob ); + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + + /* Set nByte to the number of bytes required to store the expanded blob. */ + nByte = pMem->n + pMem->u.nZero; + if( nByte<=0 ){ + nByte = 1; + } + if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ + return SQLITE_NOMEM; + } + + memset(&pMem->z[pMem->n], 0, pMem->u.nZero); + pMem->n += pMem->u.nZero; + pMem->flags &= ~(MEM_Zero|MEM_Term); } + return SQLITE_OK; } +#endif -#if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) /* -** Print a single opcode. This routine is used for debugging only. +** Make sure the given Mem is \u0000 terminated. */ -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ - char *zP4; - char zPtr[50]; - static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-4s %.2X %s\n"; - if( pOut==0 ) pOut = stdout; - zP4 = displayP4(pOp, zPtr, sizeof(zPtr)); - fprintf(pOut, zFormat1, pc, - sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, zP4, pOp->p5, -#ifdef SQLITE_DEBUG - pOp->zComment ? pOp->zComment : "" -#else - "" -#endif - ); - fflush(pOut); +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( (pMem->flags & MEM_Term)!=0 || (pMem->flags & MEM_Str)==0 ){ + return SQLITE_OK; /* Nothing to do */ + } + if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ + return SQLITE_NOMEM; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->flags |= MEM_Term; + return SQLITE_OK; } -#endif /* -** Release an array of N Mem elements -*/ -static void releaseMemArray(Mem *p, int N){ - if( p && N ){ - Mem *pEnd; - sqlite3 *db = p->db; - u8 malloc_failed = db->mallocFailed; - if( db->pnBytesFreed ){ - for(pEnd=&p[N]; pzMalloc); - } - return; - } - for(pEnd=&p[N]; pflags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ - sqlite3VdbeMemRelease(p); - }else if( p->zMalloc ){ - sqlite3DbFree(db, p->zMalloc); - p->zMalloc = 0; - } - - p->flags = MEM_Null; - } - db->mallocFailed = malloc_failed; - } -} - -/* -** Delete a VdbeFrame object and its contents. VdbeFrame objects are -** allocated by the OP_Program opcode in sqlite3VdbeExec(). -*/ -SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ - int i; - Mem *aMem = VdbeFrameMem(p); - VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; - for(i=0; inChildCsr; i++){ - sqlite3VdbeFreeCursor(p->v, apCsr[i]); - } - releaseMemArray(aMem, p->nChildMem); - sqlite3DbFree(p->v->db, p); -} - -#ifndef SQLITE_OMIT_EXPLAIN -/* -** Give a listing of the program in the virtual machine. -** -** The interface is the same as sqlite3VdbeExec(). But instead of -** running the code, it invokes the callback once for each instruction. -** This feature is used to implement "EXPLAIN". +** Add MEM_Str to the set of representations for the given Mem. Numbers +** are converted using sqlite3_snprintf(). Converting a BLOB to a string +** is a no-op. ** -** When p->explain==1, each instruction is listed. When -** p->explain==2, only OP_Explain instructions are listed and these -** are shown in a different format. p->explain==2 is used to implement -** EXPLAIN QUERY PLAN. +** Existing representations MEM_Int and MEM_Real are *not* invalidated. ** -** When p->explain==1, first the main program is listed, then each of -** the trigger subprograms are listed one by one. +** A MEM_Null value will never be passed to this function. This function is +** used for converting values to text for returning to the user (i.e. via +** sqlite3_value_text()), or for ensuring that values to be used as btree +** keys are strings. In the former case a NULL pointer is returned the +** user and the later is an internal programming error. */ -SQLITE_PRIVATE int sqlite3VdbeList( - Vdbe *p /* The VDBE */ -){ - int nRow; /* Stop when row count reaches this */ - int nSub = 0; /* Number of sub-vdbes seen so far */ - SubProgram **apSub = 0; /* Array of sub-vdbes */ - Mem *pSub = 0; /* Memory cell hold array of subprogs */ - sqlite3 *db = p->db; /* The database connection */ - int i; /* Loop counter */ - int rc = SQLITE_OK; /* Return code */ - Mem *pMem = p->pResultSet = &p->aMem[1]; /* First Mem of result set */ +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, int enc){ + int rc = SQLITE_OK; + int fg = pMem->flags; + const int nByte = 32; - assert( p->explain ); - assert( p->magic==VDBE_MAGIC_RUN ); - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( !(fg&MEM_Zero) ); + assert( !(fg&(MEM_Str|MEM_Blob)) ); + assert( fg&(MEM_Int|MEM_Real) ); + assert( (pMem->flags&MEM_RowSet)==0 ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - /* Even though this opcode does not use dynamic strings for - ** the result, result columns may become dynamic if the user calls - ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. - */ - releaseMemArray(pMem, 8); - if( p->rc==SQLITE_NOMEM ){ - /* This happens if a malloc() inside a call to sqlite3_column_text() or - ** sqlite3_column_text16() failed. */ - db->mallocFailed = 1; - return SQLITE_ERROR; + if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + return SQLITE_NOMEM; } - /* When the number of output rows reaches nRow, that means the - ** listing has finished and sqlite3_step() should return SQLITE_DONE. - ** nRow is the sum of the number of rows in the main program, plus - ** the sum of the number of rows in all trigger subprograms encountered - ** so far. The nRow value will increase as new trigger subprograms are - ** encountered, but p->pc will eventually catch up to nRow. + /* For a Real or Integer, use sqlite3_mprintf() to produce the UTF-8 + ** string representation of the value. Then, if the required encoding + ** is UTF-16le or UTF-16be do a translation. + ** + ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. */ - nRow = p->nOp; - if( p->explain==1 ){ - /* The first 8 memory cells are used for the result set. So we will - ** commandeer the 9th cell to use as storage for an array of pointers - ** to trigger subprograms. The VDBE is guaranteed to have at least 9 - ** cells. */ - assert( p->nMem>9 ); - pSub = &p->aMem[9]; - if( pSub->flags&MEM_Blob ){ - /* On the first call to sqlite3_step(), pSub will hold a NULL. It is - ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ - nSub = pSub->n/sizeof(Vdbe*); - apSub = (SubProgram **)pSub->z; - } - for(i=0; inOp; - } - } - - do{ - i = p->pc++; - }while( iexplain==2 && p->aOp[i].opcode!=OP_Explain ); - if( i>=nRow ){ - p->rc = SQLITE_OK; - rc = SQLITE_DONE; - }else if( db->u1.isInterrupted ){ - p->rc = SQLITE_INTERRUPT; - rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); + if( fg & MEM_Int ){ + sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); }else{ - char *z; - Op *pOp; - if( inOp ){ - /* The output line number is small enough that we are still in the - ** main program. */ - pOp = &p->aOp[i]; - }else{ - /* We are currently listing subprograms. Figure out which one and - ** pick up the appropriate opcode. */ - int j; - i -= p->nOp; - for(j=0; i>=apSub[j]->nOp; j++){ - i -= apSub[j]->nOp; - } - pOp = &apSub[j]->aOp[i]; - } - if( p->explain==1 ){ - pMem->flags = MEM_Int; - pMem->type = SQLITE_INTEGER; - pMem->u.i = i; /* Program counter */ - pMem++; - - pMem->flags = MEM_Static|MEM_Str|MEM_Term; - pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ - assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30(pMem->z); - pMem->type = SQLITE_TEXT; - pMem->enc = SQLITE_UTF8; - pMem++; - - /* When an OP_Program opcode is encounter (the only opcode that has - ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms - ** kept in p->aMem[9].z to hold the new program - assuming this subprogram - ** has not already been seen. - */ - if( pOp->p4type==P4_SUBPROGRAM ){ - int nByte = (nSub+1)*sizeof(SubProgram*); - int j; - for(j=0; jp4.pProgram ) break; - } - if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, 1) ){ - apSub = (SubProgram **)pSub->z; - apSub[nSub++] = pOp->p4.pProgram; - pSub->flags |= MEM_Blob; - pSub->n = nSub*sizeof(SubProgram*); - } - } - } - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p1; /* P1 */ - pMem->type = SQLITE_INTEGER; - pMem++; - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p2; /* P2 */ - pMem->type = SQLITE_INTEGER; - pMem++; - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p3; /* P3 */ - pMem->type = SQLITE_INTEGER; - pMem++; - - if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; - z = displayP4(pOp, pMem->z, 32); - if( z!=pMem->z ){ - sqlite3VdbeMemSetStr(pMem, z, -1, SQLITE_UTF8, 0); - }else{ - assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - } - pMem->type = SQLITE_TEXT; - pMem++; - - if( p->explain==1 ){ - if( sqlite3VdbeMemGrow(pMem, 4, 0) ){ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; - pMem->n = 2; - sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ - pMem->type = SQLITE_TEXT; - pMem->enc = SQLITE_UTF8; - pMem++; - -#ifdef SQLITE_DEBUG - if( pOp->zComment ){ - pMem->flags = MEM_Str|MEM_Term; - pMem->z = pOp->zComment; - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - pMem->type = SQLITE_TEXT; - }else -#endif - { - pMem->flags = MEM_Null; /* Comment */ - pMem->type = SQLITE_NULL; - } - } - - p->nResColumn = 8 - 4*(p->explain-1); - p->rc = SQLITE_OK; - rc = SQLITE_ROW; + assert( fg & MEM_Real ); + sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->r); } + pMem->n = sqlite3Strlen30(pMem->z); + pMem->enc = SQLITE_UTF8; + pMem->flags |= MEM_Str|MEM_Term; + sqlite3VdbeChangeEncoding(pMem, enc); return rc; } -#endif /* SQLITE_OMIT_EXPLAIN */ -#ifdef SQLITE_DEBUG /* -** Print the SQL that was used to generate a VDBE program. +** Memory cell pMem contains the context of an aggregate function. +** This routine calls the finalize method for that function. The +** result of the aggregate is stored back into pMem. +** +** Return SQLITE_ERROR if the finalizer reports an error. SQLITE_OK +** otherwise. */ -SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe *p){ - int nOp = p->nOp; - VdbeOp *pOp; - if( nOp<1 ) return; - pOp = &p->aOp[0]; - if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ - const char *z = pOp->p4.z; - while( sqlite3Isspace(*z) ) z++; - printf("SQL: [%s]\n", z); +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ + int rc = SQLITE_OK; + if( ALWAYS(pFunc && pFunc->xFinalize) ){ + sqlite3_context ctx; + assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + ctx.s.flags = MEM_Null; + ctx.s.db = pMem->db; + ctx.pMem = pMem; + ctx.pFunc = pFunc; + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ + assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); + sqlite3DbFree(pMem->db, pMem->zMalloc); + memcpy(pMem, &ctx.s, sizeof(ctx.s)); + rc = ctx.isError; } + return rc; } -#endif -#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* -** Print an IOTRACE message showing SQL content. +** If the memory cell contains a string value that must be freed by +** invoking an external callback, free it now. Calling this function +** does not free any Mem.zMalloc buffer. */ -SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ - int nOp = p->nOp; - VdbeOp *pOp; - if( sqlite3IoTrace==0 ) return; - if( nOp<1 ) return; - pOp = &p->aOp[0]; - if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ - int i, j; - char z[1000]; - sqlite3_snprintf(sizeof(z), z, "%s", pOp->p4.z); - for(i=0; sqlite3Isspace(z[i]); i++){} - for(j=0; z[i]; i++){ - if( sqlite3Isspace(z[i]) ){ - if( z[i-1]!=' ' ){ - z[j++] = ' '; - } - }else{ - z[j++] = z[i]; - } +SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p){ + assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); + testcase( p->flags & MEM_Agg ); + testcase( p->flags & MEM_Dyn ); + testcase( p->flags & MEM_RowSet ); + testcase( p->flags & MEM_Frame ); + if( p->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame) ){ + if( p->flags&MEM_Agg ){ + sqlite3VdbeMemFinalize(p, p->u.pDef); + assert( (p->flags & MEM_Agg)==0 ); + sqlite3VdbeMemRelease(p); + }else if( p->flags&MEM_Dyn && p->xDel ){ + assert( (p->flags&MEM_RowSet)==0 ); + p->xDel((void *)p->z); + p->xDel = 0; + }else if( p->flags&MEM_RowSet ){ + sqlite3RowSetClear(p->u.pRowSet); + }else if( p->flags&MEM_Frame ){ + sqlite3VdbeMemSetNull(p); } - z[j] = 0; - sqlite3IoTrace("SQL %s\n", z); } } -#endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ /* -** Allocate space from a fixed size buffer and return a pointer to -** that space. If insufficient space is available, return NULL. -** -** The pBuf parameter is the initial value of a pointer which will -** receive the new memory. pBuf is normally NULL. If pBuf is not -** NULL, it means that memory space has already been allocated and that -** this routine should not allocate any new memory. When pBuf is not -** NULL simply return pBuf. Only allocate new memory space when pBuf -** is NULL. -** -** nByte is the number of bytes of space needed. -** -** *ppFrom points to available space and pEnd points to the end of the -** available space. When space is allocated, *ppFrom is advanced past -** the end of the allocated space. -** -** *pnByte is a counter of the number of bytes of space that have failed -** to allocate. If there is insufficient space in *ppFrom to satisfy the -** request, then increment *pnByte by the amount of the request. +** Release any memory held by the Mem. This may leave the Mem in an +** inconsistent state, for example with (Mem.z==0) and +** (Mem.type==SQLITE_TEXT). */ -static void *allocSpace( - void *pBuf, /* Where return pointer will be stored */ - int nByte, /* Number of bytes to allocate */ - u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ - u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ - int *pnByte /* If allocation cannot be made, increment *pnByte */ -){ - assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); - if( pBuf ) return pBuf; - nByte = ROUND8(nByte); - if( &(*ppFrom)[nByte] <= pEnd ){ - pBuf = (void*)*ppFrom; - *ppFrom += nByte; - }else{ - *pnByte += nByte; - } - return pBuf; +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ + sqlite3VdbeMemReleaseExternal(p); + sqlite3DbFree(p->db, p->zMalloc); + p->z = 0; + p->zMalloc = 0; + p->xDel = 0; } /* -** Prepare a virtual machine for execution. This involves things such -** as allocating stack space and initializing the program counter. -** After the VDBE has be prepped, it can be executed by one or more -** calls to sqlite3VdbeExec(). -** -** This is the only way to move a VDBE from VDBE_MAGIC_INIT to -** VDBE_MAGIC_RUN. +** Convert a 64-bit IEEE double into a 64-bit signed integer. +** If the double is too large, return 0x8000000000000000. ** -** This function may be called more than once on a single virtual machine. -** The first call is made while compiling the SQL statement. Subsequent -** calls are made as part of the process of resetting a statement to be -** re-executed (from a call to sqlite3_reset()). The nVar, nMem, nCursor -** and isExplain parameters are only passed correct values the first time -** the function is called. On subsequent calls, from sqlite3_reset(), nVar -** is passed -1 and nMem, nCursor and isExplain are all passed zero. +** Most systems appear to do this simply by assigning +** variables and without the extra range tests. But +** there are reports that windows throws an expection +** if the floating point value is out of range. (See ticket #2880.) +** Because we do not completely understand the problem, we will +** take the conservative approach and always do range tests +** before attempting the conversion. */ -SQLITE_PRIVATE void sqlite3VdbeMakeReady( - Vdbe *p, /* The VDBE */ - int nVar, /* Number of '?' see in the SQL statement */ - int nMem, /* Number of memory cells to allocate */ - int nCursor, /* Number of cursors to allocate */ - int nArg, /* Maximum number of args in SubPrograms */ - int isExplain, /* True if the EXPLAIN keywords is present */ - int usesStmtJournal /* True to set Vdbe.usesStmtJournal */ -){ - int n; - sqlite3 *db = p->db; - - assert( p!=0 ); - assert( p->magic==VDBE_MAGIC_INIT ); - - /* There should be at least one opcode. - */ - assert( p->nOp>0 ); - - /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ - p->magic = VDBE_MAGIC_RUN; - - /* For each cursor required, also allocate a memory cell. Memory - ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by - ** the vdbe program. Instead they are used to allocate space for - ** VdbeCursor/BtCursor structures. The blob of memory associated with - ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) - ** stores the blob of memory associated with cursor 1, etc. - ** - ** See also: allocateCursor(). +static i64 doubleToInt64(double r){ +#ifdef SQLITE_OMIT_FLOATING_POINT + /* When floating-point is omitted, double and int64 are the same thing */ + return r; +#else + /* + ** Many compilers we encounter do not define constants for the + ** minimum and maximum 64-bit integers, or they define them + ** inconsistently. And many do not understand the "LL" notation. + ** So we define our own static constants here using nothing + ** larger than a 32-bit integer constant. */ - nMem += nCursor; - - /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. This is only done the - ** first time this function is called for a given VDBE, not when it is - ** being called from sqlite3_reset() to reset the virtual machine. - */ - if( nVar>=0 && ALWAYS(db->mallocFailed==0) ){ - u8 *zCsr = (u8 *)&p->aOp[p->nOp]; /* Memory avaliable for alloation */ - u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; /* First byte past available mem */ - int nByte; /* How much extra memory needed */ - - resolveP2Values(p, &nArg); - p->usesStmtJournal = (u8)usesStmtJournal; - if( isExplain && nMem<10 ){ - nMem = 10; - } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); - - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the - ** end of the opcode array. If we are unable to satisfy all memory - ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. - ** - ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly - ** reduce the amount of memory held by a prepared statement. - */ - do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); - - p->nCursor = (u16)nCursor; - if( p->aVar ){ - p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Null; - p->aMem[n].db = db; - } - } - } -#ifdef SQLITE_DEBUG - for(n=1; nnMem; n++){ - assert( p->aMem[n].db==db ); - } -#endif + static const i64 maxInt = LARGEST_INT64; + static const i64 minInt = SMALLEST_INT64; - p->pc = -1; - p->rc = SQLITE_OK; - p->errorAction = OE_Abort; - p->explain |= isExplain; - p->magic = VDBE_MAGIC_RUN; - p->nChange = 0; - p->cacheCtr = 1; - p->minWriteFileFormat = 255; - p->iStatement = 0; - p->nFkConstraint = 0; -#ifdef VDBE_PROFILE - { - int i; - for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; - } + if( r<(double)minInt ){ + return minInt; + }else if( r>(double)maxInt ){ + /* minInt is correct here - not maxInt. It turns out that assigning + ** a very large positive number to an integer results in a very large + ** negative integer. This makes no sense, but it is what x86 hardware + ** does so for compatibility we will do the same in software. */ + return minInt; + }else{ + return (i64)r; } #endif } /* -** Close a VDBE cursor and release all the resources that cursor -** happens to hold. +** Return some kind of integer value which is the best we can do +** at representing the value that *pMem describes as an integer. +** If pMem is an integer, then the value is exact. If pMem is +** a floating-point then the value returned is the integer part. +** If pMem is a string or blob, then we make an attempt to convert +** it into a integer and return that. If pMem represents an +** an SQL-NULL value, return 0. +** +** If pMem represents a string value, its encoding might be changed. */ -SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ - if( pCx==0 ){ - return; - } - if( pCx->pBt ){ - sqlite3BtreeClose(pCx->pBt); - /* The pCx->pCursor will be close automatically, if it exists, by - ** the call above. */ - }else if( pCx->pCursor ){ - sqlite3BtreeCloseCursor(pCx->pCursor); - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pCx->pVtabCursor ){ - sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; - const sqlite3_module *pModule = pCx->pModule; - p->inVtabMethod = 1; - pModule->xClose(pVtabCursor); - p->inVtabMethod = 0; +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ + int flags; + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + flags = pMem->flags; + if( flags & MEM_Int ){ + return pMem->u.i; + }else if( flags & MEM_Real ){ + return doubleToInt64(pMem->r); + }else if( flags & (MEM_Str|MEM_Blob) ){ + i64 value = 0; + assert( pMem->z || pMem->n==0 ); + testcase( pMem->z==0 ); + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); + return value; + }else{ + return 0; } -#endif } /* -** Copy the values stored in the VdbeFrame structure to its Vdbe. This -** is used, for example, when a trigger sub-program is halted to restore -** control to the main program. +** Return the best representation of pMem that we can get into a +** double. If pMem is already a double or an integer, return its +** value. If it is a string or blob, try to convert it to a double. +** If it is a NULL, return 0.0. */ -SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ - Vdbe *v = pFrame->v; - v->aOp = pFrame->aOp; - v->nOp = pFrame->nOp; - v->aMem = pFrame->aMem; - v->nMem = pFrame->nMem; - v->apCsr = pFrame->apCsr; - v->nCursor = pFrame->nCursor; - v->db->lastRowid = pFrame->lastRowid; - v->nChange = pFrame->nChange; - return pFrame->pc; +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + if( pMem->flags & MEM_Real ){ + return pMem->r; + }else if( pMem->flags & MEM_Int ){ + return (double)pMem->u.i; + }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + double val = (double)0; + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); + return val; + }else{ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + return (double)0; + } } /* -** Close all cursors. -** -** Also release any dynamic memory held by the VM in the Vdbe.aMem memory -** cell array. This is necessary as the memory cell array may contain -** pointers to VdbeFrame objects, which may in turn contain pointers to -** open cursors. +** The MEM structure is already a MEM_Real. Try to also make it a +** MEM_Int if we can. */ -static void closeAllCursors(Vdbe *p){ - if( p->pFrame ){ - VdbeFrame *pFrame = p->pFrame; - for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); - sqlite3VdbeFrameRestore(pFrame); +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ + assert( pMem->flags & MEM_Real ); + assert( (pMem->flags & MEM_RowSet)==0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->u.i = doubleToInt64(pMem->r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. On x86 hardware, the third term is always + ** true and could be omitted. But we leave it in because other + ** architectures might behave differently. + */ + if( pMem->r==(double)pMem->u.i && pMem->u.i>SMALLEST_INT64 + && ALWAYS(pMem->u.iflags |= MEM_Int; } - p->pFrame = 0; - p->nFrame = 0; +} - if( p->apCsr ){ - int i; - for(i=0; inCursor; i++){ - VdbeCursor *pC = p->apCsr[i]; - if( pC ){ - sqlite3VdbeFreeCursor(p, pC); - p->apCsr[i] = 0; - } +/* +** Convert pMem to type integer. Invalidate any prior representations. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( (pMem->flags & MEM_RowSet)==0 ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->u.i = sqlite3VdbeIntValue(pMem); + MemSetTypeFlag(pMem, MEM_Int); + return SQLITE_OK; +} + +/* +** Convert pMem so that it is of type MEM_Real. +** Invalidate any prior representations. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + return SQLITE_OK; +} + +/* +** Convert pMem so that it has types MEM_Real or MEM_Int or both. +** Invalidate any prior representations. +** +** Every effort is made to force the conversion, even if the input +** is a string that does not look completely like a number. Convert +** as much of the string as we can and ignore the rest. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ + MemSetTypeFlag(pMem, MEM_Int); + }else{ + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + sqlite3VdbeIntegerAffinity(pMem); } } - if( p->aMem ){ - releaseMemArray(&p->aMem[1], p->nMem); + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob); + return SQLITE_OK; +} + +/* +** Delete any previous value and set the value stored in *pMem to NULL. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ + if( pMem->flags & MEM_Frame ){ + VdbeFrame *pFrame = pMem->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; } - while( p->pDelFrame ){ - VdbeFrame *pDel = p->pDelFrame; - p->pDelFrame = pDel->pParent; - sqlite3VdbeFrameDelete(pDel); + if( pMem->flags & MEM_RowSet ){ + sqlite3RowSetClear(pMem->u.pRowSet); } + MemSetTypeFlag(pMem, MEM_Null); + pMem->type = SQLITE_NULL; } /* -** Clean up the VM after execution. -** -** This routine will automatically close any cursors, lists, and/or -** sorters that were left open. It also deletes the values of -** variables in the aVar[] array. +** Delete any previous value and set the value to be a BLOB of length +** n containing all zeros. */ -static void Cleanup(Vdbe *p){ - sqlite3 *db = p->db; +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ + sqlite3VdbeMemRelease(pMem); + pMem->flags = MEM_Blob|MEM_Zero; + pMem->type = SQLITE_BLOB; + pMem->n = 0; + if( n<0 ) n = 0; + pMem->u.nZero = n; + pMem->enc = SQLITE_UTF8; -#ifdef SQLITE_DEBUG - /* Execute assert() statements to ensure that the Vdbe.apCsr[] and - ** Vdbe.aMem[] arrays have already been cleaned up. */ - int i; - for(i=0; inCursor; i++) assert( p->apCsr==0 || p->apCsr[i]==0 ); - for(i=1; i<=p->nMem; i++) assert( p->aMem==0 || p->aMem[i].flags==MEM_Null ); +#ifdef SQLITE_OMIT_INCRBLOB + sqlite3VdbeMemGrow(pMem, n, 0); + if( pMem->z ){ + pMem->n = n; + memset(pMem->z, 0, n); + } #endif +} - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - p->pResultSet = 0; +/* +** Delete any previous value and set the value stored in *pMem to val, +** manifest type INTEGER. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ + sqlite3VdbeMemRelease(pMem); + pMem->u.i = val; + pMem->flags = MEM_Int; + pMem->type = SQLITE_INTEGER; } +#ifndef SQLITE_OMIT_FLOATING_POINT /* -** Set the number of result columns that will be returned by this SQL -** statement. This is now set at compile time, rather than during -** execution of the vdbe program so that sqlite3_column_count() can -** be called on an SQL statement before sqlite3_step(). +** Delete any previous value and set the value stored in *pMem to val, +** manifest type REAL. */ -SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ - Mem *pColName; - int n; - sqlite3 *db = p->db; +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ + if( sqlite3IsNaN(val) ){ + sqlite3VdbeMemSetNull(pMem); + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->r = val; + pMem->flags = MEM_Real; + pMem->type = SQLITE_FLOAT; + } +} +#endif - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - sqlite3DbFree(db, p->aColName); - n = nResColumn*COLNAME_N; - p->nResColumn = (u16)nResColumn; - p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n ); - if( p->aColName==0 ) return; - while( n-- > 0 ){ - pColName->flags = MEM_Null; - pColName->db = p->db; - pColName++; +/* +** Delete any previous value and set the value of pMem to be an +** empty boolean index. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ + sqlite3 *db = pMem->db; + assert( db!=0 ); + assert( (pMem->flags & MEM_RowSet)==0 ); + sqlite3VdbeMemRelease(pMem); + pMem->zMalloc = sqlite3DbMallocRaw(db, 64); + if( db->mallocFailed ){ + pMem->flags = MEM_Null; + }else{ + assert( pMem->zMalloc ); + pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, + sqlite3DbMallocSize(db, pMem->zMalloc)); + assert( pMem->u.pRowSet!=0 ); + pMem->flags = MEM_RowSet; } } /* -** Set the name of the idx'th column to be returned by the SQL statement. -** zName must be a pointer to a nul terminated string. -** -** This call must be made after a call to sqlite3VdbeSetNumCols(). -** -** The final parameter, xDel, must be one of SQLITE_DYNAMIC, SQLITE_STATIC -** or SQLITE_TRANSIENT. If it is SQLITE_DYNAMIC, then the buffer pointed -** to by zName will be freed by sqlite3DbFree() when the vdbe is destroyed. +** Return true if the Mem object contains a TEXT or BLOB that is +** too large - whose size exceeds SQLITE_MAX_LENGTH. */ -SQLITE_PRIVATE int sqlite3VdbeSetColName( - Vdbe *p, /* Vdbe being configured */ - int idx, /* Index of column zName applies to */ - int var, /* One of the COLNAME_* constants */ - const char *zName, /* Pointer to buffer containing name */ - void (*xDel)(void*) /* Memory management strategy for zName */ -){ - int rc; - Mem *pColName; - assert( idxnResColumn ); - assert( vardb->mallocFailed ){ - assert( !zName || xDel!=SQLITE_DYNAMIC ); - return SQLITE_NOMEM; +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ + assert( p->db!=0 ); + if( p->flags & (MEM_Str|MEM_Blob) ){ + int n = p->n; + if( p->flags & MEM_Zero ){ + n += p->u.nZero; + } + return n>p->db->aLimit[SQLITE_LIMIT_LENGTH]; } - assert( p->aColName!=0 ); - pColName = &(p->aColName[idx+var*p->nResColumn]); - rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); - assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); - return rc; + return 0; } +#ifdef SQLITE_DEBUG /* -** A read or write transaction may or may not be active on database handle -** db. If a transaction is active, commit it. If there is a -** write-transaction spanning more than one database file, this routine -** takes care of the master journal trickery. +** This routine prepares a memory cell for modication by breaking +** its link to a shallow copy and by marking any current shallow +** copies of this cell as invalid. +** +** This is used for testing and debugging only - to make sure shallow +** copies are not misused. */ -static int vdbeCommit(sqlite3 *db, Vdbe *p){ +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ int i; - int nTrans = 0; /* Number of databases with an active write-transaction */ - int rc = SQLITE_OK; - int needXcommit = 0; + Mem *pX; + for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + pX->flags |= MEM_Invalid; + pX->pScopyFrom = 0; + } + } + pMem->pScopyFrom = 0; +} +#endif /* SQLITE_DEBUG */ -#ifdef SQLITE_OMIT_VIRTUALTABLE - /* With this option, sqlite3VtabSync() is defined to be simply - ** SQLITE_OK so p is not used. - */ - UNUSED_PARAMETER(p); -#endif +/* +** Size of struct Mem not including the Mem.zMalloc member. +*/ +#define MEMCELLSIZE (size_t)(&(((Mem *)0)->zMalloc)) - /* Before doing anything else, call the xSync() callback for any - ** virtual module tables written in this transaction. This has to - ** be done before determining whether a master journal file is - ** required, as an xSync() callback may add an attached database - ** to the transaction. - */ - rc = sqlite3VtabSync(db, &p->zErrMsg); +/* +** Make an shallow copy of pFrom into pTo. Prior contents of +** pTo are freed. The pFrom->z field is not duplicated. If +** pFrom->z is used, then pTo->z points to the same thing as pFrom->z +** and flags gets srcType (either MEM_Ephem or MEM_Static). +*/ +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ + assert( (pFrom->flags & MEM_RowSet)==0 ); + sqlite3VdbeMemReleaseExternal(pTo); + memcpy(pTo, pFrom, MEMCELLSIZE); + pTo->xDel = 0; + if( (pFrom->flags&MEM_Static)==0 ){ + pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); + assert( srcType==MEM_Ephem || srcType==MEM_Static ); + pTo->flags |= srcType; + } +} - /* This loop determines (a) if the commit hook should be invoked and - ** (b) how many database files have open write transactions, not - ** including the temp database. (b) is important because if more than - ** one database file has an open write transaction, a master journal - ** file is required for an atomic commit. - */ - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( sqlite3BtreeIsInTrans(pBt) ){ - needXcommit = 1; - if( i!=1 ) nTrans++; - rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); +/* +** Make a full copy of pFrom into pTo. Prior contents of pTo are +** freed before the copy is made. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ + int rc = SQLITE_OK; + + assert( (pFrom->flags & MEM_RowSet)==0 ); + sqlite3VdbeMemReleaseExternal(pTo); + memcpy(pTo, pFrom, MEMCELLSIZE); + pTo->flags &= ~MEM_Dyn; + + if( pTo->flags&(MEM_Str|MEM_Blob) ){ + if( 0==(pFrom->flags&MEM_Static) ){ + pTo->flags |= MEM_Ephem; + rc = sqlite3VdbeMemMakeWriteable(pTo); } } - if( rc!=SQLITE_OK ){ - return rc; + + return rc; +} + +/* +** Transfer the contents of pFrom to pTo. Any existing value in pTo is +** freed. If pFrom contains ephemeral data, a copy is made. +** +** pFrom contains an SQL NULL when this routine returns. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ + assert( pFrom->db==0 || sqlite3_mutex_held(pFrom->db->mutex) ); + assert( pTo->db==0 || sqlite3_mutex_held(pTo->db->mutex) ); + assert( pFrom->db==0 || pTo->db==0 || pFrom->db==pTo->db ); + + sqlite3VdbeMemRelease(pTo); + memcpy(pTo, pFrom, sizeof(Mem)); + pFrom->flags = MEM_Null; + pFrom->xDel = 0; + pFrom->zMalloc = 0; +} + +/* +** Change the value of a Mem to be a string or a BLOB. +** +** The memory management strategy depends on the value of the xDel +** parameter. If the value passed is SQLITE_TRANSIENT, then the +** string is copied into a (possibly existing) buffer managed by the +** Mem structure. Otherwise, any existing buffer is freed and the +** pointer copied. +** +** If the string is too large (if it exceeds the SQLITE_LIMIT_LENGTH +** size limit) then no memory allocation occurs. If the string can be +** stored without allocating memory, then it is. If a memory allocation +** is required to store the string, then value of pMem is unchanged. In +** either case, SQLITE_TOOBIG is returned. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemSetStr( + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ +){ + int nByte = n; /* New value for pMem->n */ + int iLimit; /* Maximum allowed string or blob size */ + u16 flags = 0; /* New value for pMem->flags */ + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( (pMem->flags & MEM_RowSet)==0 ); + + /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ + if( !z ){ + sqlite3VdbeMemSetNull(pMem); + return SQLITE_OK; } - /* If there are any write-transactions at all, invoke the commit hook */ - if( needXcommit && db->xCommitCallback ){ - rc = db->xCommitCallback(db->pCommitArg); - if( rc ){ - return SQLITE_CONSTRAINT; + if( pMem->db ){ + iLimit = pMem->db->aLimit[SQLITE_LIMIT_LENGTH]; + }else{ + iLimit = SQLITE_MAX_LENGTH; + } + flags = (enc==0?MEM_Blob:MEM_Str); + if( nByte<0 ){ + assert( enc!=0 ); + if( enc==SQLITE_UTF8 ){ + for(nByte=0; nByte<=iLimit && z[nByte]; nByte++){} + }else{ + for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } + flags |= MEM_Term; } - /* The simple case - no more than one database file (not counting the - ** TEMP database) has a transaction active. There is no need for the - ** master-journal. - ** - ** If the return value of sqlite3BtreeGetFilename() is a zero length - ** string, it means the main database is :memory: or a temp file. In - ** that case we do not support atomic multi-file commits, so use the - ** simple case then too. + /* The following block sets the new values of Mem.z and Mem.xDel. It + ** also sets a flag in local variable "flags" to indicate the memory + ** management (one of MEM_Dyn or MEM_Static). */ - if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) - || nTrans<=1 - ){ - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, 0); - } + if( xDel==SQLITE_TRANSIENT ){ + int nAlloc = nByte; + if( flags&MEM_Term ){ + nAlloc += (enc==SQLITE_UTF8?1:2); } - - /* Do the commit only if all databases successfully complete phase 1. - ** If one of the BtreeCommitPhaseOne() calls fails, this indicates an - ** IO error while deleting or truncating a journal file. It is unlikely, - ** but could happen. In this case abandon processing and return the error. - */ - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseTwo(pBt); - } + if( nByte>iLimit ){ + return SQLITE_TOOBIG; } - if( rc==SQLITE_OK ){ - sqlite3VtabCommit(db); + if( sqlite3VdbeMemGrow(pMem, nAlloc, 0) ){ + return SQLITE_NOMEM; } - } - - /* The complex case - There is a multi-file write-transaction active. - ** This requires a master journal file to ensure the transaction is - ** committed atomicly. - */ -#ifndef SQLITE_OMIT_DISKIO - else{ - sqlite3_vfs *pVfs = db->pVfs; - int needSync = 0; - char *zMaster = 0; /* File-name for the master journal */ - char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); - sqlite3_file *pMaster = 0; - i64 offset = 0; - int res; + memcpy(pMem->z, z, nAlloc); + }else if( xDel==SQLITE_DYNAMIC ){ + sqlite3VdbeMemRelease(pMem); + pMem->zMalloc = pMem->z = (char *)z; + pMem->xDel = 0; + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->z = (char *)z; + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } - /* Select a master journal file name */ - do { - u32 iRandom; - sqlite3DbFree(db, zMaster); - sqlite3_randomness(sizeof(iRandom), &iRandom); - zMaster = sqlite3MPrintf(db, "%s-mj%08X", zMainFile, iRandom&0x7fffffff); - if( !zMaster ){ - return SQLITE_NOMEM; - } - rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); - }while( rc==SQLITE_OK && res ); - if( rc==SQLITE_OK ){ - /* Open the master journal. */ - rc = sqlite3OsOpenMalloc(pVfs, zMaster, &pMaster, - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_MASTER_JOURNAL, 0 - ); + pMem->n = nByte; + pMem->flags = flags; + pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); + pMem->type = (enc==0 ? SQLITE_BLOB : SQLITE_TEXT); + +#ifndef SQLITE_OMIT_UTF16 + if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + return SQLITE_NOMEM; + } +#endif + + if( nByte>iLimit ){ + return SQLITE_TOOBIG; + } + + return SQLITE_OK; +} + +/* +** Compare the values contained by the two memory cells, returning +** negative, zero or positive if pMem1 is less than, equal to, or greater +** than pMem2. Sorting order is NULL's first, followed by numbers (integers +** and reals) sorted numerically, followed by text ordered by the collating +** sequence pColl and finally blob's ordered by memcmp(). +** +** Two NULL values are considered equal by this function. +*/ +SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ + int rc; + int f1, f2; + int combined_flags; + + f1 = pMem1->flags; + f2 = pMem2->flags; + combined_flags = f1|f2; + assert( (combined_flags & MEM_RowSet)==0 ); + + /* If one value is NULL, it is less than the other. If both values + ** are NULL, return 0. + */ + if( combined_flags&MEM_Null ){ + return (f2&MEM_Null) - (f1&MEM_Null); + } + + /* If one value is a number and the other is not, the number is less. + ** If both are numbers, compare as reals if one is a real, or as integers + ** if both values are integers. + */ + if( combined_flags&(MEM_Int|MEM_Real) ){ + if( !(f1&(MEM_Int|MEM_Real)) ){ + return 1; } - if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, zMaster); - return rc; + if( !(f2&(MEM_Int|MEM_Real)) ){ + return -1; } - - /* Write the name of each database file in the transaction into the new - ** master journal file. If an error occurs at this point close - ** and delete the master journal file. All the individual journal files - ** still have 'null' as the master journal pointer, so they will roll - ** back independently if a failure occurs. - */ - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( sqlite3BtreeIsInTrans(pBt) ){ - char const *zFile = sqlite3BtreeGetJournalname(pBt); - if( zFile==0 ){ - continue; /* Ignore TEMP and :memory: databases */ - } - assert( zFile[0]!=0 ); - if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ - needSync = 1; - } - rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); - offset += sqlite3Strlen30(zFile)+1; - if( rc!=SQLITE_OK ){ - sqlite3OsCloseFree(pMaster); - sqlite3OsDelete(pVfs, zMaster, 0); - sqlite3DbFree(db, zMaster); - return rc; - } + if( (f1 & f2 & MEM_Int)==0 ){ + double r1, r2; + if( (f1&MEM_Real)==0 ){ + r1 = (double)pMem1->u.i; + }else{ + r1 = pMem1->r; } + if( (f2&MEM_Real)==0 ){ + r2 = (double)pMem2->u.i; + }else{ + r2 = pMem2->r; + } + if( r1r2 ) return 1; + return 0; + }else{ + assert( f1&MEM_Int ); + assert( f2&MEM_Int ); + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return 1; + return 0; } + } - /* Sync the master journal file. If the IOCAP_SEQUENTIAL device - ** flag is set this is not required. - */ - if( needSync - && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) - && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) - ){ - sqlite3OsCloseFree(pMaster); - sqlite3OsDelete(pVfs, zMaster, 0); - sqlite3DbFree(db, zMaster); - return rc; - } - - /* Sync all the db files involved in the transaction. The same call - ** sets the master journal pointer in each individual journal. If - ** an error occurs here, do not delete the master journal file. - ** - ** If the error occurs during the first call to - ** sqlite3BtreeCommitPhaseOne(), then there is a chance that the - ** master journal file will be orphaned. But we cannot delete it, - ** in case the master journal file name was written into the journal - ** file before the failure occurred. - */ - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, zMaster); - } + /* If one value is a string and the other is a blob, the string is less. + ** If both are strings, compare using the collating functions. + */ + if( combined_flags&MEM_Str ){ + if( (f1 & MEM_Str)==0 ){ + return 1; } - sqlite3OsCloseFree(pMaster); - assert( rc!=SQLITE_BUSY ); - if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, zMaster); - return rc; + if( (f2 & MEM_Str)==0 ){ + return -1; } - /* Delete the master journal file. This commits the transaction. After - ** doing this the directory is synced again before any individual - ** transaction files are deleted. - */ - rc = sqlite3OsDelete(pVfs, zMaster, 1); - sqlite3DbFree(db, zMaster); - zMaster = 0; - if( rc ){ - return rc; - } + assert( pMem1->enc==pMem2->enc ); + assert( pMem1->enc==SQLITE_UTF8 || + pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); - /* All files and directories have already been synced, so the following - ** calls to sqlite3BtreeCommitPhaseTwo() are only closing files and - ** deleting or truncating journals. If something goes wrong while - ** this is happening we don't really care. The integrity of the - ** transaction is already guaranteed, but some stray 'cold' journals - ** may be lying around. Returning an error code won't help matters. + /* The collation sequence must be defined at this point, even if + ** the user deletes the collation sequence after the vdbe program is + ** compiled (this was not always the case). */ - disable_simulated_io_errors(); - sqlite3BeginBenignMalloc(); - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - sqlite3BtreeCommitPhaseTwo(pBt); + assert( !pColl || pColl->xCmp ); + + if( pColl ){ + if( pMem1->enc==pColl->enc ){ + /* The strings are already in the correct encoding. Call the + ** comparison function directly */ + return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); + }else{ + const void *v1, *v2; + int n1, n2; + Mem c1; + Mem c2; + memset(&c1, 0, sizeof(c1)); + memset(&c2, 0, sizeof(c2)); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + n1 = v1==0 ? 0 : c1.n; + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + n2 = v2==0 ? 0 : c2.n; + rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + sqlite3VdbeMemRelease(&c1); + sqlite3VdbeMemRelease(&c2); + return rc; } } - sqlite3EndBenignMalloc(); - enable_simulated_io_errors(); - - sqlite3VtabCommit(db); + /* If a NULL pointer was passed as the collate function, fall through + ** to the blob case and use memcmp(). */ + } + + /* Both values must be blobs. Compare using memcmp(). */ + rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); + if( rc==0 ){ + rc = pMem1->n - pMem2->n; } -#endif - return rc; } -/* -** This routine checks that the sqlite3.activeVdbeCnt count variable -** matches the number of vdbe's in the list sqlite3.pVdbe that are -** currently active. An assertion fails if the two counts do not match. -** This is an internal self-check only - it is not an essential processing -** step. +/* +** Move data out of a btree key or data field and into a Mem structure. +** The data or key is taken from the entry that pCur is currently pointing +** to. offset and amt determine what portion of the data or key to retrieve. +** key is true to get the key or false to get data. The result is written +** into the pMem element. ** -** This is a no-op if NDEBUG is defined. +** The pMem structure is assumed to be uninitialized. Any prior content +** is overwritten without being freed. +** +** If this routine fails for any reason (malloc returns NULL or unable +** to read from the disk) then the pMem is left in an inconsistent state. */ -#ifndef NDEBUG -static void checkActiveVdbeCnt(sqlite3 *db){ - Vdbe *p; - int cnt = 0; - int nWrite = 0; - p = db->pVdbe; - while( p ){ - if( p->magic==VDBE_MAGIC_RUN && p->pc>=0 ){ - cnt++; - if( p->readOnly==0 ) nWrite++; - } - p = p->pNext; +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + int offset, /* Offset from the start of data to return bytes from. */ + int amt, /* Number of bytes to return. */ + int key, /* If true, retrieve from the btree key, not data. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + char *zData; /* Data from the btree layer */ + int available = 0; /* Number of bytes available on the local btree page */ + int rc = SQLITE_OK; /* Return code */ + + assert( sqlite3BtreeCursorIsValid(pCur) ); + + /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() + ** that both the BtShared and database handle mutexes are held. */ + assert( (pMem->flags & MEM_RowSet)==0 ); + if( key ){ + zData = (char *)sqlite3BtreeKeyFetch(pCur, &available); + }else{ + zData = (char *)sqlite3BtreeDataFetch(pCur, &available); } - assert( cnt==db->activeVdbeCnt ); - assert( nWrite==db->writeVdbeCnt ); -} -#else -#define checkActiveVdbeCnt(x) -#endif + assert( zData!=0 ); -/* -** For every Btree that in database connection db which -** has been modified, "trip" or invalidate each cursor in -** that Btree might have been modified so that the cursor -** can never be used again. This happens when a rollback -*** occurs. We have to trip all the other cursors, even -** cursor from other VMs in different database connections, -** so that none of them try to use the data at which they -** were pointing and which now may have been changed due -** to the rollback. -** -** Remember that a rollback can delete tables complete and -** reorder rootpages. So it is not sufficient just to save -** the state of the cursor. We have to invalidate the cursor -** so that it is never used again. -*/ -static void invalidateCursorsOnModifiedBtrees(sqlite3 *db){ - int i; - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p && sqlite3BtreeIsInTrans(p) ){ - sqlite3BtreeTripAllCursors(p, SQLITE_ABORT); + if( offset+amt<=available && (pMem->flags&MEM_Dyn)==0 ){ + sqlite3VdbeMemRelease(pMem); + pMem->z = &zData[offset]; + pMem->flags = MEM_Blob|MEM_Ephem; + }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ + pMem->flags = MEM_Blob|MEM_Dyn|MEM_Term; + pMem->enc = 0; + pMem->type = SQLITE_BLOB; + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + if( rc!=SQLITE_OK ){ + sqlite3VdbeMemRelease(pMem); } } + pMem->n = amt; + + return rc; } -/* -** If the Vdbe passed as the first argument opened a statement-transaction, -** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or -** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement -** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the -** statement transaction is commtted. +/* This function is only available internally, it is not part of the +** external API. It works in a similar way to sqlite3_value_text(), +** except the data returned is in the encoding specified by the second +** parameter, which must be one of SQLITE_UTF16BE, SQLITE_UTF16LE or +** SQLITE_UTF8. ** -** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. -** Otherwise SQLITE_OK. +** (2006-02-16:) The enc value can be or-ed with SQLITE_UTF16_ALIGNED. +** If that is the case, then the result must be aligned on an even byte +** boundary. */ -SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ - sqlite3 *const db = p->db; - int rc = SQLITE_OK; - - /* If p->iStatement is greater than zero, then this Vdbe opened a - ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occured, causing an emergency rollback. - ** In this case (db->nStatement==0), and there is nothing to do. - */ - if( db->nStatement && p->iStatement ){ - int i; - const int iSavepoint = p->iStatement-1; +SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ + if( !pVal ) return 0; - assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); - assert( db->nStatement>0 ); - assert( p->iStatement==(db->nStatement+db->nSavepoint) ); + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( (pVal->flags & MEM_RowSet)==0 ); - for(i=0; inDb; i++){ - int rc2 = SQLITE_OK; - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( eOp==SAVEPOINT_ROLLBACK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); - } - if( rc2==SQLITE_OK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); - } - if( rc==SQLITE_OK ){ - rc = rc2; - } + if( pVal->flags&MEM_Null ){ + return 0; + } + assert( (MEM_Blob>>3) == MEM_Str ); + pVal->flags |= (pVal->flags & MEM_Blob)>>3; + expandBlob(pVal); + if( pVal->flags&MEM_Str ){ + sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); + if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ + assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); + if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ + return 0; } } - db->nStatement--; - p->iStatement = 0; - - /* If the statement transaction is being rolled back, also restore the - ** database handles deferred constraint counter to the value it had when - ** the statement transaction was opened. */ - if( eOp==SAVEPOINT_ROLLBACK ){ - db->nDeferredCons = p->nStmtDefCons; - } + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-59893-45467 */ + }else{ + assert( (pVal->flags&MEM_Blob)==0 ); + sqlite3VdbeMemStringify(pVal, enc); + assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + } + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + return pVal->z; + }else{ + return 0; } - return rc; } /* -** If SQLite is compiled to support shared-cache mode and to be threadsafe, -** this routine obtains the mutex associated with each BtShared structure -** that may be accessed by the VM passed as an argument. In doing so it -** sets the BtShared.db member of each of the BtShared structures, ensuring -** that the correct busy-handler callback is invoked if required. -** -** If SQLite is not threadsafe but does support shared-cache mode, then -** sqlite3BtreeEnterAll() is invoked to set the BtShared.db variables -** of all of BtShared structures accessible via the database handle -** associated with the VM. Of course only a subset of these structures -** will be accessed by the VM, and we could use Vdbe.btreeMask to figure -** that subset out, but there is no advantage to doing so. -** -** If SQLite is not threadsafe and does not support shared-cache mode, this -** function is a no-op. +** Create a new sqlite3_value object. */ -#ifndef SQLITE_OMIT_SHARED_CACHE -SQLITE_PRIVATE void sqlite3VdbeMutexArrayEnter(Vdbe *p){ -#if SQLITE_THREADSAFE - sqlite3BtreeMutexArrayEnter(&p->aMutex); -#else - sqlite3BtreeEnterAll(p->db); -#endif +SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ + Mem *p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p ){ + p->flags = MEM_Null; + p->type = SQLITE_NULL; + p->db = db; + } + return p; } -#endif /* -** This function is called when a transaction opened by the database -** handle associated with the VM passed as an argument is about to be -** committed. If there are outstanding deferred foreign key constraint -** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** Create a new sqlite3_value object, containing the value of pExpr. ** -** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT and write -** an error message to it. Then return SQLITE_ERROR. +** This only works for very simple expressions that consist of one constant +** token (i.e. "5", "5.1", "'a string'"). If the expression can +** be converted directly into a value, then the value is allocated and +** a pointer written to *ppVal. The caller is responsible for deallocating +** the value by passing it to sqlite3ValueFree() later on. If the expression +** cannot be converted to a value, then *ppVal is set to NULL. */ -#ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ - sqlite3 *db = p->db; - if( (deferred && db->nDeferredCons>0) || (!deferred && p->nFkConstraint>0) ){ - p->rc = SQLITE_CONSTRAINT; - p->errorAction = OE_Abort; - sqlite3SetString(&p->zErrMsg, db, "foreign key constraint failed"); - return SQLITE_ERROR; +SQLITE_PRIVATE int sqlite3ValueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* Write the new value here */ +){ + int op; + char *zVal = 0; + sqlite3_value *pVal = 0; + int negInt = 1; + const char *zNeg = ""; + + if( !pExpr ){ + *ppVal = 0; + return SQLITE_OK; } - return SQLITE_OK; -} + op = pExpr->op; + + /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2. + ** The ifdef here is to enable us to achieve 100% branch test coverage even + ** when SQLITE_ENABLE_STAT2 is omitted. + */ +#ifdef SQLITE_ENABLE_STAT2 + if( op==TK_REGISTER ) op = pExpr->op2; +#else + if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; #endif -/* -** This routine is called the when a VDBE tries to halt. If the VDBE -** has made changes and is in autocommit mode, then commit those -** changes. If a rollback is needed, then do the rollback. -** -** This routine is the only way to move the state of a VM from -** SQLITE_MAGIC_RUN to SQLITE_MAGIC_HALT. It is harmless to -** call this on a VM that is in the SQLITE_MAGIC_HALT state. -** -** Return an error code. If the commit could not complete because of -** lock contention, return SQLITE_BUSY. If SQLITE_BUSY is returned, it -** means the close did not happen and needs to be repeated. -*/ -SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ - int rc; /* Used to store transient return codes */ - sqlite3 *db = p->db; - - /* This function contains the logic that determines if a statement or - ** transaction will be committed or rolled back as a result of the - ** execution of this virtual machine. - ** - ** If any of the following errors occur: - ** - ** SQLITE_NOMEM - ** SQLITE_IOERR - ** SQLITE_FULL - ** SQLITE_INTERRUPT - ** - ** Then the internal cache might have been left in an inconsistent - ** state. We need to rollback the statement transaction, if there is - ** one, or the complete transaction if there is no statement transaction. + /* Handle negative integers in a single step. This is needed in the + ** case when the value is -9223372036854775808. */ - - if( p->db->mallocFailed ){ - p->rc = SQLITE_NOMEM; - } - closeAllCursors(p); - if( p->magic!=VDBE_MAGIC_RUN ){ - return SQLITE_OK; + if( op==TK_UMINUS + && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ + pExpr = pExpr->pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; } - checkActiveVdbeCnt(db); - - /* No commit or rollback needed if the program never started */ - if( p->pc>=0 ){ - int mrc; /* Primary error code from p->rc */ - int eStatementOp = 0; - int isSpecialError; /* Set to true if a 'special' error */ - - /* Lock all btrees used by the statement */ - sqlite3VdbeMutexArrayEnter(p); - - /* Check for one of the special errors */ - mrc = p->rc & 0xff; - assert( p->rc!=SQLITE_IOERR_BLOCKED ); /* This error no longer exists */ - isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR - || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; - if( isSpecialError ){ - /* If the query was read-only and the error code is SQLITE_INTERRUPT, - ** no rollback is necessary. Otherwise, at least a savepoint - ** transaction must be rolled back to restore the database to a - ** consistent state. - ** - ** Even if the statement is read-only, it is important to perform - ** a statement or transaction rollback operation. If the error - ** occured while writing to the journal, sub-journal or database - ** file as part of an effort to free up cache space (see function - ** pagerStress() in pager.c), the rollback is required to restore - ** the pager to a consistent state. - */ - if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ - if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ - eStatementOp = SAVEPOINT_ROLLBACK; - }else{ - /* We are forced to roll back the active transaction. Before doing - ** so, abort any other statements this handle currently has active. - */ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); - sqlite3CloseSavepoints(db); - db->autoCommit = 1; - } - } - } - /* Check for immediate foreign key violations. */ - if( p->rc==SQLITE_OK ){ - sqlite3VdbeCheckFk(p, 0); + if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ + pVal = sqlite3ValueNew(db); + if( pVal==0 ) goto no_mem; + if( ExprHasProperty(pExpr, EP_IntValue) ){ + sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT; } - - /* If the auto-commit flag is set and this is the only active writer - ** VM, then we do either a commit or rollback of the current transaction. - ** - ** Note: This block also runs if one of the special errors handled - ** above has occurred. - */ - if( !sqlite3VtabInSync(db) - && db->autoCommit - && db->writeVdbeCnt==(p->readOnly==0) - ){ - if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - if( sqlite3VdbeCheckFk(p, 1) ){ - sqlite3BtreeMutexArrayLeave(&p->aMutex); - return SQLITE_ERROR; - } - /* The auto-commit flag is true, the vdbe program was successful - ** or hit an 'OR FAIL' constraint and there are no deferred foreign - ** key constraints to hold up the transaction. This means a commit - ** is required. */ - rc = vdbeCommit(db, p); - if( rc==SQLITE_BUSY ){ - sqlite3BtreeMutexArrayLeave(&p->aMutex); - return SQLITE_BUSY; - }else if( rc!=SQLITE_OK ){ - p->rc = rc; - sqlite3RollbackAll(db); - }else{ - db->nDeferredCons = 0; - sqlite3CommitInternalChanges(db); - } - }else{ - sqlite3RollbackAll(db); - } - db->nStatement = 0; - }else if( eStatementOp==0 ){ - if( p->rc==SQLITE_OK || p->errorAction==OE_Fail ){ - eStatementOp = SAVEPOINT_RELEASE; - }else if( p->errorAction==OE_Abort ){ - eStatementOp = SAVEPOINT_ROLLBACK; - }else{ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); - sqlite3CloseSavepoints(db); - db->autoCommit = 1; - } + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + }else{ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } - - /* If eStatementOp is non-zero, then a statement transaction needs to - ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to - ** do so. If this operation returns an error, and the current statement - ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the - ** current statement error code. - ** - ** Note that sqlite3VdbeCloseStatement() can only fail if eStatementOp - ** is SAVEPOINT_ROLLBACK. But if p->rc==SQLITE_OK then eStatementOp - ** must be SAVEPOINT_RELEASE. Hence the NEVER(p->rc==SQLITE_OK) in - ** the following code. - */ - if( eStatementOp ){ - rc = sqlite3VdbeCloseStatement(p, eStatementOp); - if( rc ){ - assert( eStatementOp==SAVEPOINT_ROLLBACK ); - if( NEVER(p->rc==SQLITE_OK) || p->rc==SQLITE_CONSTRAINT ){ - p->rc = rc; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - } - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); - sqlite3CloseSavepoints(db); - db->autoCommit = 1; - } + if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; + if( enc!=SQLITE_UTF8 ){ + sqlite3VdbeChangeEncoding(pVal, enc); } - - /* If this was an INSERT, UPDATE or DELETE and no statement transaction - ** has been rolled back, update the database connection change-counter. - */ - if( p->changeCntOn ){ - if( eStatementOp!=SAVEPOINT_ROLLBACK ){ - sqlite3VdbeSetChanges(db, p->nChange); + }else if( op==TK_UMINUS ) { + /* This branch happens for multiple negative signs. Ex: -(-5) */ + if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){ + sqlite3VdbeMemNumerify(pVal); + if( pVal->u.i==SMALLEST_INT64 ){ + pVal->flags &= MEM_Int; + pVal->flags |= MEM_Real; + pVal->r = (double)LARGEST_INT64; }else{ - sqlite3VdbeSetChanges(db, 0); + pVal->u.i = -pVal->u.i; } - p->nChange = 0; - } - - /* Rollback or commit any schema changes that occurred. */ - if( p->rc!=SQLITE_OK && db->flags&SQLITE_InternChanges ){ - sqlite3ResetInternalSchema(db, 0); - db->flags = (db->flags | SQLITE_InternChanges); - } - - /* Release the locks */ - sqlite3BtreeMutexArrayLeave(&p->aMutex); - } - - /* We have successfully halted and closed the VM. Record this fact. */ - if( p->pc>=0 ){ - db->activeVdbeCnt--; - if( !p->readOnly ){ - db->writeVdbeCnt--; + pVal->r = -pVal->r; + sqlite3ValueApplyAffinity(pVal, affinity, enc); } - assert( db->activeVdbeCnt>=db->writeVdbeCnt ); + }else if( op==TK_NULL ){ + pVal = sqlite3ValueNew(db); + if( pVal==0 ) goto no_mem; } - p->magic = VDBE_MAGIC_HALT; - checkActiveVdbeCnt(db); - if( p->db->mallocFailed ){ - p->rc = SQLITE_NOMEM; +#ifndef SQLITE_OMIT_BLOB_LITERAL + else if( op==TK_BLOB ){ + int nVal; + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + pVal = sqlite3ValueNew(db); + if( !pVal ) goto no_mem; + zVal = &pExpr->u.zToken[2]; + nVal = sqlite3Strlen30(zVal)-1; + assert( zVal[nVal]=='\'' ); + sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2, + 0, SQLITE_DYNAMIC); } +#endif - /* If the auto-commit flag is set to true, then any locks that were held - ** by connection db have now been released. Call sqlite3ConnectionUnlocked() - ** to invoke any required unlock-notify callbacks. - */ - if( db->autoCommit ){ - sqlite3ConnectionUnlocked(db); + if( pVal ){ + sqlite3VdbeMemStoreType(pVal); } - - assert( db->activeVdbeCnt>0 || db->autoCommit==0 || db->nStatement==0 ); + *ppVal = pVal; return SQLITE_OK; + +no_mem: + db->mallocFailed = 1; + sqlite3DbFree(db, zVal); + sqlite3ValueFree(pVal); + *ppVal = 0; + return SQLITE_NOMEM; } +/* +** Change the string value of an sqlite3_value object +*/ +SQLITE_PRIVATE void sqlite3ValueSetStr( + sqlite3_value *v, /* Value to be set */ + int n, /* Length of string z */ + const void *z, /* Text of the new string */ + u8 enc, /* Encoding to use */ + void (*xDel)(void*) /* Destructor for the string */ +){ + if( v ) sqlite3VdbeMemSetStr((Mem *)v, z, n, enc, xDel); +} /* -** Each VDBE holds the result of the most recent sqlite3_step() call -** in p->rc. This routine sets that result back to SQLITE_OK. +** Free an sqlite3_value object */ -SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe *p){ - p->rc = SQLITE_OK; +SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ + if( !v ) return; + sqlite3VdbeMemRelease((Mem *)v); + sqlite3DbFree(((Mem*)v)->db, v); } /* -** Clean up a VDBE after execution but do not delete the VDBE just yet. -** Write any error messages into *pzErrMsg. Return the result code. +** Return the number of bytes in the sqlite3_value object assuming +** that it uses the encoding "enc" +*/ +SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ + Mem *p = (Mem*)pVal; + if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ + if( p->flags & MEM_Zero ){ + return p->n + p->u.nZero; + }else{ + return p->n; + } + } + return 0; +} + +/************** End of vdbemem.c *********************************************/ +/************** Begin file vdbeaux.c *****************************************/ +/* +** 2003 September 6 ** -** After this routine is run, the VDBE should be ready to be executed -** again. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** To look at it another way, this routine resets the state of the -** virtual machine from VDBE_MAGIC_RUN or VDBE_MAGIC_HALT back to -** VDBE_MAGIC_INIT. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used for creating, destroying, and populating +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior +** to version 2.8.7, all this code was combined into the vdbe.c source file. +** But that file was getting too big so this subroutines were split out. */ -SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ - sqlite3 *db; - db = p->db; - /* If the VM did not run to completion or if it encountered an - ** error, then it might not have been halted properly. So halt - ** it now. - */ - sqlite3VdbeHalt(p); - /* If the VDBE has be run even partially, then transfer the error code - ** and error message from the VDBE into the main database structure. But - ** if the VDBE has just been set to run but has not actually executed any - ** instructions yet, leave the main database error information unchanged. - */ - if( p->pc>=0 ){ - if( p->zErrMsg ){ - sqlite3BeginBenignMalloc(); - sqlite3ValueSetStr(db->pErr,-1,p->zErrMsg,SQLITE_UTF8,SQLITE_TRANSIENT); - sqlite3EndBenignMalloc(); - db->errCode = p->rc; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - }else if( p->rc ){ - sqlite3Error(db, p->rc, 0); - }else{ - sqlite3Error(db, SQLITE_OK, 0); - } - if( p->runOnlyOnce ) p->expired = 1; - }else if( p->rc && p->expired ){ - /* The expired flag was set on the VDBE before the first call - ** to sqlite3_step(). For consistency (since sqlite3_step() was - ** called), set the database error in this case as well. - */ - sqlite3Error(db, p->rc, 0); - sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - } - /* Reclaim all memory used by the VDBE - */ - Cleanup(p); +/* +** When debugging the code generator in a symbolic debugger, one can +** set the sqlite3VdbeAddopTrace to 1 and all opcodes will be printed +** as they are added to the instruction stream. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeAddopTrace = 0; +#endif - /* Save profiling information from this VDBE run. - */ -#ifdef VDBE_PROFILE - { - FILE *out = fopen("vdbe_profile.out", "a"); - if( out ){ - int i; - fprintf(out, "---- "); - for(i=0; inOp; i++){ - fprintf(out, "%02x", p->aOp[i].opcode); - } - fprintf(out, "\n"); - for(i=0; inOp; i++){ - fprintf(out, "%6d %10lld %8lld ", - p->aOp[i].cnt, - p->aOp[i].cycles, - p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 - ); - sqlite3VdbePrintOp(out, i, &p->aOp[i]); - } - fclose(out); - } + +/* +** Create a new virtual database engine. +*/ +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(sqlite3 *db){ + Vdbe *p; + p = sqlite3DbMallocZero(db, sizeof(Vdbe) ); + if( p==0 ) return 0; + p->db = db; + if( db->pVdbe ){ + db->pVdbe->pPrev = p; } -#endif + p->pNext = db->pVdbe; + p->pPrev = 0; + db->pVdbe = p; p->magic = VDBE_MAGIC_INIT; - return p->rc & db->errMask; + return p; } - + /* -** Clean up and delete a VDBE after execution. Return an integer which is -** the result code. Write any error message text into *pzErrMsg. +** Remember the SQL string for a prepared statement. */ -SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ - int rc = SQLITE_OK; - if( p->magic==VDBE_MAGIC_RUN || p->magic==VDBE_MAGIC_HALT ){ - rc = sqlite3VdbeReset(p); - assert( (rc & p->db->errMask)==rc ); - } - sqlite3VdbeDelete(p); - return rc; +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ + assert( isPrepareV2==1 || isPrepareV2==0 ); + if( p==0 ) return; +#ifdef SQLITE_OMIT_TRACE + if( !isPrepareV2 ) return; +#endif + assert( p->zSql==0 ); + p->zSql = sqlite3DbStrNDup(p->db, z, n); + p->isPrepareV2 = (u8)isPrepareV2; } /* -** Call the destructor for each auxdata entry in pVdbeFunc for which -** the corresponding bit in mask is clear. Auxdata entries beyond 31 -** are always destroyed. To destroy all auxdata entries, call this -** routine with mask==0. +** Return the SQL associated with a prepared statement */ -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ - int i; - for(i=0; inAux; i++){ - struct AuxData *pAux = &pVdbeFunc->apAux[i]; - if( (i>31 || !(mask&(((u32)1)<pAux ){ - if( pAux->xDelete ){ - pAux->xDelete(pAux->pAux); - } - pAux->pAux = 0; - } - } +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + return (p && p->isPrepareV2) ? p->zSql : 0; } /* -** Free all memory associated with the Vdbe passed as the second argument. -** The difference between this function and sqlite3VdbeDelete() is that -** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with -** the database connection. +** Swap all content between two VDBE structures. */ -SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ - SubProgram *pSub, *pNext; - assert( p->db==0 || p->db==db ); - releaseMemArray(p->aVar, p->nVar); - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - for(pSub=p->pProgram; pSub; pSub=pNext){ - pNext = pSub->pNext; - vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); - sqlite3DbFree(db, pSub); - } - vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->aLabel); - sqlite3DbFree(db, p->aColName); - sqlite3DbFree(db, p->zSql); - sqlite3DbFree(db, p->pFree); - sqlite3DbFree(db, p); +SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ + Vdbe tmp, *pTmp; + char *zTmp; + tmp = *pA; + *pA = *pB; + *pB = tmp; + pTmp = pA->pNext; + pA->pNext = pB->pNext; + pB->pNext = pTmp; + pTmp = pA->pPrev; + pA->pPrev = pB->pPrev; + pB->pPrev = pTmp; + zTmp = pA->zSql; + pA->zSql = pB->zSql; + pB->zSql = zTmp; + pB->isPrepareV2 = pA->isPrepareV2; } +#ifdef SQLITE_DEBUG /* -** Delete an entire VDBE. +** Turn tracing on or off */ -SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ - sqlite3 *db; +SQLITE_PRIVATE void sqlite3VdbeTrace(Vdbe *p, FILE *trace){ + p->trace = trace; +} +#endif - if( NEVER(p==0) ) return; - db = p->db; - if( p->pPrev ){ - p->pPrev->pNext = p->pNext; - }else{ - assert( db->pVdbe==p ); - db->pVdbe = p->pNext; - } - if( p->pNext ){ - p->pNext->pPrev = p->pPrev; +/* +** Resize the Vdbe.aOp array so that it is at least one op larger than +** it was. +** +** If an out-of-memory error occurs while resizing the array, return +** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain +** unchanged (this is so that any opcodes already allocated can be +** correctly deallocated along with the rest of the Vdbe). +*/ +static int growOpArray(Vdbe *p){ + VdbeOp *pNew; + int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + pNew = sqlite3DbRealloc(p->db, p->aOp, nNew*sizeof(Op)); + if( pNew ){ + p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); + p->aOp = pNew; } - p->magic = VDBE_MAGIC_DEAD; - p->db = 0; - sqlite3VdbeDeleteObject(db, p); + return (pNew ? SQLITE_OK : SQLITE_NOMEM); } /* -** Make sure the cursor p is ready to read or write the row to which it -** was last positioned. Return an error code if an OOM fault or I/O error -** prevents us from positioning the cursor to its correct position. +** Add a new instruction to the list of instructions current in the +** VDBE. Return the address of the new instruction. ** -** If a MoveTo operation is pending on the given cursor, then do that -** MoveTo now. If no move is pending, check to see if the row has been -** deleted out from under the cursor and if it has, mark the row as -** a NULL row. +** Parameters: ** -** If the cursor is already pointing to the correct row and that row has -** not been deleted out from under the cursor, then this routine is a no-op. +** p Pointer to the VDBE +** +** op The opcode for this instruction +** +** p1, p2, p3 Operands +** +** Use the sqlite3VdbeResolveLabel() function to fix an address and +** the sqlite3VdbeChangeP4() function to change the value of the P4 +** operand. */ -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ - if( p->deferredMoveto ){ - int res, rc; -#ifdef SQLITE_TEST - extern int sqlite3_search_count; -#endif - assert( p->isTable ); - rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); - if( rc ) return rc; - p->lastRowid = p->movetoTarget; - if( res!=0 ) return SQLITE_CORRUPT_BKPT; - p->rowidIsValid = 1; -#ifdef SQLITE_TEST - sqlite3_search_count++; -#endif - p->deferredMoveto = 0; - p->cacheStatus = CACHE_STALE; - }else if( ALWAYS(p->pCursor) ){ - int hasMoved; - int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); - if( rc ) return rc; - if( hasMoved ){ - p->cacheStatus = CACHE_STALE; - p->nullRow = 1; +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ + int i; + VdbeOp *pOp; + + i = p->nOp; + assert( p->magic==VDBE_MAGIC_INIT ); + assert( op>0 && op<0xff ); + if( p->nOpAlloc<=i ){ + if( growOpArray(p) ){ + return 1; } } - return SQLITE_OK; + p->nOp++; + pOp = &p->aOp[i]; + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; +#ifdef SQLITE_DEBUG + pOp->zComment = 0; + if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); +#endif +#ifdef VDBE_PROFILE + pOp->cycles = 0; + pOp->cnt = 0; +#endif + return i; +} +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ + return sqlite3VdbeAddOp3(p, op, 0, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ + return sqlite3VdbeAddOp3(p, op, p1, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ + return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } + /* -** The following functions: -** -** sqlite3VdbeSerialType() -** sqlite3VdbeSerialTypeLen() -** sqlite3VdbeSerialLen() -** sqlite3VdbeSerialPut() -** sqlite3VdbeSerialGet() -** -** encapsulate the code that serializes values for storage in SQLite -** data and index records. Each serialized value consists of a -** 'serial-type' and a blob of data. The serial type is an 8-byte unsigned -** integer, stored as a varint. -** -** In an SQLite index record, the serial type is stored directly before -** the blob of data that it corresponds to. In a table record, all serial -** types are stored at the start of the record, and the blobs of data at -** the end. Hence these functions allow the caller to handle the -** serial-type and data blob seperately. -** -** The following table describes the various storage classes for data: -** -** serial type bytes of data type -** -------------- --------------- --------------- -** 0 0 NULL -** 1 1 signed integer -** 2 2 signed integer -** 3 3 signed integer -** 4 4 signed integer -** 5 6 signed integer -** 6 8 signed integer -** 7 8 IEEE float -** 8 0 Integer constant 0 -** 9 0 Integer constant 1 -** 10,11 reserved for expansion -** N>=12 and even (N-12)/2 BLOB -** N>=13 and odd (N-13)/2 text +** Add an opcode that includes the p4 value as a pointer. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const char *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + sqlite3VdbeChangeP4(p, addr, zP4, p4type); + return addr; +} + +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also local all btrees. ** -** The 8 and 9 types were added in 3.3.0, file format 4. Prior versions -** of SQLite will not understand those serial types. +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. */ +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ + int j; + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); +} /* -** Return the serial-type for the value stored in pMem. +** Add an opcode that includes the p4 value as an integer. */ -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ - int flags = pMem->flags; - int n; +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32); + return addr; +} - if( flags&MEM_Null ){ - return 0; - } - if( flags&MEM_Int ){ - /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ -# define MAX_6BYTE ((((i64)0x00008000)<<32)-1) - i64 i = pMem->u.i; - u64 u; - if( file_format>=4 && (i&1)==i ){ - return 8+(u32)i; - } - u = i<0 ? -i : i; - if( u<=127 ) return 1; - if( u<=32767 ) return 2; - if( u<=8388607 ) return 3; - if( u<=2147483647 ) return 4; - if( u<=MAX_6BYTE ) return 5; - return 6; - } - if( flags&MEM_Real ){ - return 7; +/* +** Create a new symbolic label for an instruction that has yet to be +** coded. The symbolic label is really just a negative number. The +** label can be used as the P2 value of an operation. Later, when +** the label is resolved to a specific address, the VDBE will scan +** through its operation list and change all values of P2 which match +** the label into the resolved address. +** +** The VDBE knows that a P2 value is a label because labels are +** always negative and P2 values are suppose to be non-negative. +** Hence, a negative P2 value is a label that has yet to be resolved. +** +** Zero is returned if a malloc() fails. +*/ +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *p){ + int i; + i = p->nLabel++; + assert( p->magic==VDBE_MAGIC_INIT ); + if( i>=p->nLabelAlloc ){ + int n = p->nLabelAlloc*2 + 5; + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + n*sizeof(p->aLabel[0])); + p->nLabelAlloc = sqlite3DbMallocSize(p->db, p->aLabel)/sizeof(p->aLabel[0]); } - assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); - n = pMem->n; - if( flags & MEM_Zero ){ - n += pMem->u.nZero; + if( p->aLabel ){ + p->aLabel[i] = -1; } - assert( n>=0 ); - return ((n*2) + 12 + ((flags&MEM_Str)!=0)); + return -1-i; } /* -** Return the length of the data corresponding to the supplied serial-type. +** Resolve label "x" to be the address of the next instruction to +** be inserted. The parameter "x" must have been obtained from +** a prior call to sqlite3VdbeMakeLabel(). */ -SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ - if( serial_type>=12 ){ - return (serial_type-12)/2; - }else{ - static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; - return aSize[serial_type]; +SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *p, int x){ + int j = -1-x; + assert( p->magic==VDBE_MAGIC_INIT ); + assert( j>=0 && jnLabel ); + if( p->aLabel ){ + p->aLabel[j] = p->nOp; } } /* -** If we are on an architecture with mixed-endian floating -** points (ex: ARM7) then swap the lower 4 bytes with the -** upper 4 bytes. Return the result. -** -** For most architectures, this is a no-op. -** -** (later): It is reported to me that the mixed-endian problem -** on ARM7 is an issue with GCC, not with the ARM7 chip. It seems -** that early versions of GCC stored the two words of a 64-bit -** float in the wrong order. And that error has been propagated -** ever since. The blame is not necessarily with GCC, though. -** GCC might have just copying the problem from a prior compiler. -** I am also told that newer versions of GCC that follow a different -** ABI get the byte order right. -** -** Developers using SQLite on an ARM7 should compile and run their -** application using -DSQLITE_DEBUG=1 at least once. With DEBUG -** enabled, some asserts below will ensure that the byte order of -** floating point values is correct. -** -** (2007-08-30) Frank van Vugt has studied this problem closely -** and has send his findings to the SQLite developers. Frank -** writes that some Linux kernels offer floating point hardware -** emulation that uses only 32-bit mantissas instead of a full -** 48-bits as required by the IEEE standard. (This is the -** CONFIG_FPE_FASTFPE option.) On such systems, floating point -** byte swapping becomes very complicated. To avoid problems, -** the necessary byte swapping is carried out using a 64-bit integer -** rather than a 64-bit float. Frank assures us that the code here -** works for him. We, the developers, have no way to independently -** verify this, but Frank seems to know what he is talking about -** so we trust him. +** Mark the VDBE as one that can only be run one time. */ -#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT -static u64 floatSwap(u64 in){ - union { - u64 r; - u32 i[2]; - } u; - u32 t; - - u.r = in; - t = u.i[0]; - u.i[0] = u.i[1]; - u.i[1] = t; - return u.r; +SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ + p->runOnlyOnce = 1; } -# define swapMixedEndianFloat(X) X = floatSwap(X) -#else -# define swapMixedEndianFloat(X) -#endif + +#ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* -** Write the serialized data blob for the value stored in pMem into -** buf. It is assumed that the caller has allocated sufficient space. -** Return the number of bytes written. +** The following type and function are used to iterate through all opcodes +** in a Vdbe main program and each of the sub-programs (triggers) it may +** invoke directly or indirectly. It should be used as follows: ** -** nBuf is the amount of space left in buf[]. nBuf must always be -** large enough to hold the entire field. Except, if the field is -** a blob with a zero-filled tail, then buf[] might be just the right -** size to hold everything except for the zero-filled tail. If buf[] -** is only big enough to hold the non-zero prefix, then only write that -** prefix into buf[]. But if buf[] is large enough to hold both the -** prefix and the tail then write the prefix and set the tail to all -** zeros. +** Op *pOp; +** VdbeOpIter sIter; ** -** Return the number of bytes actually written into buf[]. The number -** of bytes in the zero-filled tail is included in the return value only -** if those bytes were zeroed in buf[]. -*/ -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, int nBuf, Mem *pMem, int file_format){ - u32 serial_type = sqlite3VdbeSerialType(pMem, file_format); - u32 len; +** memset(&sIter, 0, sizeof(sIter)); +** sIter.v = v; // v is of type Vdbe* +** while( (pOp = opIterNext(&sIter)) ){ +** // Do something with pOp +** } +** sqlite3DbFree(v->db, sIter.apSub); +** +*/ +typedef struct VdbeOpIter VdbeOpIter; +struct VdbeOpIter { + Vdbe *v; /* Vdbe to iterate through the opcodes of */ + SubProgram **apSub; /* Array of subprograms */ + int nSub; /* Number of entries in apSub */ + int iAddr; /* Address of next instruction to return */ + int iSub; /* 0 = main program, 1 = first sub-program etc. */ +}; +static Op *opIterNext(VdbeOpIter *p){ + Vdbe *v = p->v; + Op *pRet = 0; + Op *aOp; + int nOp; - /* Integer and Real */ - if( serial_type<=7 && serial_type>0 ){ - u64 v; - u32 i; - if( serial_type==7 ){ - assert( sizeof(v)==sizeof(pMem->r) ); - memcpy(&v, &pMem->r, sizeof(v)); - swapMixedEndianFloat(v); + if( p->iSub<=p->nSub ){ + + if( p->iSub==0 ){ + aOp = v->aOp; + nOp = v->nOp; }else{ - v = pMem->u.i; - } - len = i = sqlite3VdbeSerialTypeLen(serial_type); - assert( len<=(u32)nBuf ); - while( i-- ){ - buf[i] = (u8)(v&0xFF); - v >>= 8; + aOp = p->apSub[p->iSub-1]->aOp; + nOp = p->apSub[p->iSub-1]->nOp; } - return len; - } + assert( p->iAddr=12 ){ - assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) - == (int)sqlite3VdbeSerialTypeLen(serial_type) ); - assert( pMem->n<=nBuf ); - len = pMem->n; - memcpy(buf, pMem->z, len); - if( pMem->flags & MEM_Zero ){ - len += pMem->u.nZero; - assert( nBuf>=0 ); - if( len > (u32)nBuf ){ - len = (u32)nBuf; + pRet = &aOp[p->iAddr]; + p->iAddr++; + if( p->iAddr==nOp ){ + p->iSub++; + p->iAddr = 0; + } + + if( pRet->p4type==P4_SUBPROGRAM ){ + int nByte = (p->nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jnSub; j++){ + if( p->apSub[j]==pRet->p4.pProgram ) break; + } + if( j==p->nSub ){ + p->apSub = sqlite3DbReallocOrFree(v->db, p->apSub, nByte); + if( !p->apSub ){ + pRet = 0; + }else{ + p->apSub[p->nSub++] = pRet->p4.pProgram; + } } - memset(&buf[pMem->n], 0, len-pMem->n); } - return len; } - /* NULL or constants 0 or 1 */ - return 0; + return pRet; } /* -** Deserialize the data blob pointed to by buf as serial type serial_type -** and store the result in pMem. Return the number of bytes read. -*/ -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( - const unsigned char *buf, /* Buffer to deserialize from */ - u32 serial_type, /* Serial type to deserialize */ - Mem *pMem /* Memory cell to write value into */ -){ - switch( serial_type ){ - case 10: /* Reserved for future use */ - case 11: /* Reserved for future use */ - case 0: { /* NULL */ - pMem->flags = MEM_Null; - break; - } - case 1: { /* 1-byte signed integer */ - pMem->u.i = (signed char)buf[0]; - pMem->flags = MEM_Int; - return 1; - } - case 2: { /* 2-byte signed integer */ - pMem->u.i = (((signed char)buf[0])<<8) | buf[1]; - pMem->flags = MEM_Int; - return 2; - } - case 3: { /* 3-byte signed integer */ - pMem->u.i = (((signed char)buf[0])<<16) | (buf[1]<<8) | buf[2]; - pMem->flags = MEM_Int; - return 3; - } - case 4: { /* 4-byte signed integer */ - pMem->u.i = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; - pMem->flags = MEM_Int; - return 4; - } - case 5: { /* 6-byte signed integer */ - u64 x = (((signed char)buf[0])<<8) | buf[1]; - u32 y = (buf[2]<<24) | (buf[3]<<16) | (buf[4]<<8) | buf[5]; - x = (x<<32) | y; - pMem->u.i = *(i64*)&x; - pMem->flags = MEM_Int; - return 6; - } - case 6: /* 8-byte signed integer */ - case 7: { /* IEEE floating point */ - u64 x; - u32 y; -#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) - /* Verify that integers and floating point values use the same - ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is - ** defined that 64-bit floating point values really are mixed - ** endian. - */ - static const u64 t1 = ((u64)0x3ff00000)<<32; - static const double r1 = 1.0; - u64 t2 = t1; - swapMixedEndianFloat(t2); - assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); -#endif +** Check if the program stored in the VM associated with pParse may +** throw an ABORT exception (causing the statement, but not entire transaction +** to be rolled back). This condition is true if the main program or any +** sub-programs contains any of the following: +** +** * OP_Halt with P1=SQLITE_CONSTRAINT and P2=OE_Abort. +** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. +** * OP_Destroy +** * OP_VUpdate +** * OP_VRename +** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** +** Then check that the value of Parse.mayAbort is true if an +** ABORT may be thrown, or false otherwise. Return true if it does +** match, or false otherwise. This function is intended to be used as +** part of an assert statement in the compiler. Similar to: +** +** assert( sqlite3VdbeAssertMayAbort(pParse->pVdbe, pParse->mayAbort) ); +*/ +SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ + int hasAbort = 0; + Op *pOp; + VdbeOpIter sIter; + memset(&sIter, 0, sizeof(sIter)); + sIter.v = v; - x = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; - y = (buf[4]<<24) | (buf[5]<<16) | (buf[6]<<8) | buf[7]; - x = (x<<32) | y; - if( serial_type==6 ){ - pMem->u.i = *(i64*)&x; - pMem->flags = MEM_Int; - }else{ - assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); - swapMixedEndianFloat(x); - memcpy(&pMem->r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; - } - return 8; - } - case 8: /* Integer 0 */ - case 9: { /* Integer 1 */ - pMem->u.i = serial_type-8; - pMem->flags = MEM_Int; - return 0; - } - default: { - u32 len = (serial_type-12)/2; - pMem->z = (char *)buf; - pMem->n = len; - pMem->xDel = 0; - if( serial_type&0x01 ){ - pMem->flags = MEM_Str | MEM_Ephem; - }else{ - pMem->flags = MEM_Blob | MEM_Ephem; - } - return len; + while( (pOp = opIterNext(&sIter))!=0 ){ + int opcode = pOp->opcode; + if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename +#ifndef SQLITE_OMIT_FOREIGN_KEY + || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) +#endif + || ((opcode==OP_Halt || opcode==OP_HaltIfNull) + && (pOp->p1==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + ){ + hasAbort = 1; + break; } } - return 0; -} + sqlite3DbFree(v->db, sIter.apSub); + /* Return true if hasAbort==mayAbort. Or if a malloc failure occured. + ** If malloc failed, then the while() loop above may not have iterated + ** through all opcodes and hasAbort may be set incorrectly. Return + ** true for this case to prevent the assert() in the callers frame + ** from failing. */ + return ( v->db->mallocFailed || hasAbort==mayAbort ); +} +#endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ /* -** Given the nKey-byte encoding of a record in pKey[], parse the -** record into a UnpackedRecord structure. Return a pointer to -** that structure. +** Loop through the program looking for P2 values that are negative +** on jump instructions. Each such value is a label. Resolve the +** label by setting the P2 value to its correct non-zero value. ** -** The calling function might provide szSpace bytes of memory -** space at pSpace. This space can be used to hold the returned -** VDbeParsedRecord structure if it is large enough. If it is -** not big enough, space is obtained from sqlite3_malloc(). +** This routine is called once after all opcodes have been inserted. ** -** The returned structure should be closed by a call to -** sqlite3VdbeDeleteUnpackedRecord(). -*/ -SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ - int nKey, /* Size of the binary record */ - const void *pKey, /* The binary record */ - char *pSpace, /* Unaligned space available to hold the object */ - int szSpace /* Size of pSpace[] in bytes */ -){ - const unsigned char *aKey = (const unsigned char *)pKey; - UnpackedRecord *p; /* The unpacked record that we will return */ - int nByte; /* Memory space needed to hold p, in bytes */ - int d; - u32 idx; - u16 u; /* Unsigned loop counter */ - u32 szHdr; - Mem *pMem; - int nOff; /* Increase pSpace by this much to 8-byte align it */ - - /* - ** We want to shift the pointer pSpace up such that it is 8-byte aligned. - ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift - ** it by. If pSpace is already 8-byte aligned, nOff should be zero. - */ - nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; - pSpace += nOff; - szSpace -= nOff; - nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); - if( nByte>szSpace ){ - p = sqlite3DbMallocRaw(pKeyInfo->db, nByte); - if( p==0 ) return 0; - p->flags = UNPACKED_NEED_FREE | UNPACKED_NEED_DESTROY; - }else{ - p = (UnpackedRecord*)pSpace; - p->flags = UNPACKED_NEED_DESTROY; - } - p->pKeyInfo = pKeyInfo; - p->nField = pKeyInfo->nField + 1; - p->aMem = pMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; - assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - idx = getVarint32(aKey, szHdr); - d = szHdr; - u = 0; - while( idxnField && d<=nKey ){ - u32 serial_type; +** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument +** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by +** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** +** The Op.opflags field is set on all opcodes. +*/ +static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ + int i; + int nMaxArgs = *pMaxFuncArgs; + Op *pOp; + int *aLabel = p->aLabel; + p->readOnly = 1; + for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ + u8 opcode = pOp->opcode; - idx += getVarint32(&aKey[idx], serial_type); - pMem->enc = pKeyInfo->enc; - pMem->db = pKeyInfo->db; - pMem->flags = 0; - pMem->zMalloc = 0; - d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); - pMem++; - u++; + pOp->opflags = sqlite3OpcodeProperty[opcode]; + if( opcode==OP_Function || opcode==OP_AggStep ){ + if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; + }else if( (opcode==OP_Transaction && pOp->p2!=0) || opcode==OP_Vacuum ){ + p->readOnly = 0; +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( opcode==OP_VUpdate ){ + if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + }else if( opcode==OP_VFilter ){ + int n; + assert( p->nOp - i >= 3 ); + assert( pOp[-1].opcode==OP_Integer ); + n = pOp[-1].p1; + if( n>nMaxArgs ) nMaxArgs = n; +#endif + } + + if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ + assert( -1-pOp->p2nLabel ); + pOp->p2 = aLabel[-1-pOp->p2]; + } } - assert( u<=pKeyInfo->nField + 1 ); - p->nField = u; - return (void*)p; + sqlite3DbFree(p->db, p->aLabel); + p->aLabel = 0; + + *pMaxFuncArgs = nMaxArgs; } /* -** This routine destroys a UnpackedRecord object. +** Return the address of the next instruction to be inserted. */ -SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ - int i; - Mem *pMem; - - assert( p!=0 ); - assert( p->flags & UNPACKED_NEED_DESTROY ); - for(i=0, pMem=p->aMem; inField; i++, pMem++){ - /* The unpacked record is always constructed by the - ** sqlite3VdbeUnpackRecord() function above, which makes all - ** strings and blobs static. And none of the elements are - ** ever transformed, so there is never anything to delete. - */ - if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); - } - if( p->flags & UNPACKED_NEED_FREE ){ - sqlite3DbFree(p->pKeyInfo->db, p); - } +SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ + assert( p->magic==VDBE_MAGIC_INIT ); + return p->nOp; } /* -** This function compares the two table rows or index records -** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero -** or positive integer if key1 is less than, equal to or -** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 -** key must be a parsed key such as obtained from -** sqlite3VdbeParseRecord. -** -** Key1 and Key2 do not have to contain the same number of fields. -** The key with fewer fields is usually compares less than the -** longer key. However if the UNPACKED_INCRKEY flags in pPKey2 is set -** and the common prefixes are equal, then key1 is less than key2. -** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are -** equal, then the keys are considered to be equal and -** the parts beyond the common prefix are ignored. +** This function returns a pointer to the array of opcodes associated with +** the Vdbe passed as the first argument. It is the callers responsibility +** to arrange for the returned array to be eventually freed using the +** vdbeFreeOpArray() function. ** -** If the UNPACKED_IGNORE_ROWID flag is set, then the last byte of -** the header of pKey1 is ignored. It is assumed that pKey1 is -** an index key, and thus ends with a rowid value. The last byte -** of the header will therefore be the serial type of the rowid: -** one of 1, 2, 3, 4, 5, 6, 8, or 9 - the integer serial types. -** The serial type of the final rowid will always be a single byte. -** By ignoring this last byte of the header, we force the comparison -** to ignore the rowid at the end of key1. +** Before returning, *pnOp is set to the number of entries in the returned +** array. Also, *pnMaxArg is set to the larger of its current value and +** the number of entries in the Vdbe.apArg[] array required to execute the +** returned program. */ -SQLITE_PRIVATE int sqlite3VdbeRecordCompare( - int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ -){ - int d1; /* Offset into aKey[] of next data element */ - u32 idx1; /* Offset into aKey[] of next header element */ - u32 szHdr1; /* Number of bytes in header */ - int i = 0; - int nField; - int rc = 0; - const unsigned char *aKey1 = (const unsigned char *)pKey1; - KeyInfo *pKeyInfo; - Mem mem1; - - pKeyInfo = pPKey2->pKeyInfo; - mem1.enc = pKeyInfo->enc; - mem1.db = pKeyInfo->db; - /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ - - /* Compilers may complain that mem1.u.i is potentially uninitialized. - ** We could initialize it, as shown here, to silence those complaints. - ** But in fact, mem1.u.i will never actually be used initialized, and doing - ** the unnecessary initialization has a measurable negative performance - ** impact, since this routine is a very high runner. And so, we choose - ** to ignore the compiler warnings and leave this variable uninitialized. - */ - /* mem1.u.i = 0; // not needed, here to silence compiler warning */ - - idx1 = getVarint32(aKey1, szHdr1); - d1 = szHdr1; - if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){ - szHdr1--; - } - nField = pKeyInfo->nField; - while( idx1nField ){ - u32 serial_type1; - - /* Read the serial types for the next element in each key. */ - idx1 += getVarint32( aKey1+idx1, serial_type1 ); - if( d1>=nKey1 && sqlite3VdbeSerialTypeLen(serial_type1)>0 ) break; +SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ + VdbeOp *aOp = p->aOp; + assert( aOp && !p->db->mallocFailed ); - /* Extract the values to be compared. - */ - d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ + assert( p->btreeMask==0 ); - /* Do the comparison - */ - rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], - iaColl[i] : 0); - if( rc!=0 ){ - assert( mem1.zMalloc==0 ); /* See comment below */ + resolveP2Values(p, pnMaxArg); + *pnOp = p->nOp; + p->aOp = 0; + return aOp; +} - /* Invert the result if we are using DESC sort order. */ - if( pKeyInfo->aSortOrder && iaSortOrder[i] ){ - rc = -rc; +/* +** Add a whole list of operations to the operation stack. Return the +** address of the first operation added. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){ + int addr; + assert( p->magic==VDBE_MAGIC_INIT ); + if( p->nOp + nOp > p->nOpAlloc && growOpArray(p) ){ + return 0; + } + addr = p->nOp; + if( ALWAYS(nOp>0) ){ + int i; + VdbeOpList const *pIn = aOp; + for(i=0; ip2; + VdbeOp *pOut = &p->aOp[i+addr]; + pOut->opcode = pIn->opcode; + pOut->p1 = pIn->p1; + if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){ + pOut->p2 = addr + ADDR(p2); + }else{ + pOut->p2 = p2; } - - /* If the PREFIX_SEARCH flag is set and all fields except the final - ** rowid field were equal, then clear the PREFIX_SEARCH flag and set - ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1). - ** This is used by the OP_IsUnique opcode. - */ - if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){ - assert( idx1==szHdr1 && rc ); - assert( mem1.flags & MEM_Int ); - pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH; - pPKey2->rowid = mem1.u.i; + pOut->p3 = pIn->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; +#ifdef SQLITE_DEBUG + pOut->zComment = 0; + if( sqlite3VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } - - return rc; +#endif } - i++; - } - - /* No memory allocation is ever used on mem1. Prove this using - ** the following assert(). If the assert() fails, it indicates a - ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). - */ - assert( mem1.zMalloc==0 ); - - /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. If the UNPACKED_INCRKEY - ** flag is set, then break the tie by treating key2 as larger. - ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes - ** are considered to be equal. Otherwise, the longer key is the - ** larger. As it happens, the pPKey2 will always be the longer - ** if there is a difference. - */ - assert( rc==0 ); - if( pPKey2->flags & UNPACKED_INCRKEY ){ - rc = -1; - }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){ - /* Leave rc==0 */ - }else if( idx1nOp += nOp; } - return rc; + return addr; } - /* -** pCur points at an index entry created using the OP_MakeRecord opcode. -** Read the rowid (the last field in the record) and store it in *rowid. -** Return SQLITE_OK if everything works, or an error code otherwise. -** -** pCur might be pointing to text obtained from a corrupt database file. -** So the content cannot be trusted. Do appropriate checks on the content. +** Change the value of the P1 operand for a specific instruction. +** This routine is useful when a large program is loaded from a +** static array using sqlite3VdbeAddOpList but we want to make a +** few minor changes to the program. */ -SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ - i64 nCellKey = 0; - int rc; - u32 szHdr; /* Size of the header */ - u32 typeRowid; /* Serial type of the rowid */ - u32 lenRowid; /* Size of the rowid */ - Mem m, v; - - UNUSED_PARAMETER(db); - - /* Get the size of the index entry. Only indices entries of less - ** than 2GiB are support - anything large must be database corruption. - ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so - ** this code can safely assume that nCellKey is 32-bits - */ - assert( sqlite3BtreeCursorIsValid(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &nCellKey); - assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ - assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); - - /* Read in the complete content of the index entry */ - memset(&m, 0, sizeof(m)); - rc = sqlite3VdbeMemFromBtree(pCur, 0, (int)nCellKey, 1, &m); - if( rc ){ - return rc; - } - - /* The index entry must begin with a header size */ - (void)getVarint32((u8*)m.z, szHdr); - testcase( szHdr==3 ); - testcase( szHdr==m.n ); - if( unlikely(szHdr<3 || (int)szHdr>m.n) ){ - goto idx_rowid_corruption; - } - - /* The last field of the index should be an integer - the ROWID. - ** Verify that the last entry really is an integer. */ - (void)getVarint32((u8*)&m.z[szHdr-1], typeRowid); - testcase( typeRowid==1 ); - testcase( typeRowid==2 ); - testcase( typeRowid==3 ); - testcase( typeRowid==4 ); - testcase( typeRowid==5 ); - testcase( typeRowid==6 ); - testcase( typeRowid==8 ); - testcase( typeRowid==9 ); - if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ - goto idx_rowid_corruption; - } - lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); - testcase( (u32)m.n==szHdr+lenRowid ); - if( unlikely((u32)m.n=0 ); + if( p->nOp>addr ){ + p->aOp[addr].p1 = val; } - - /* Fetch the integer off the end of the index record */ - sqlite3VdbeSerialGet((u8*)&m.z[m.n-lenRowid], typeRowid, &v); - *rowid = v.u.i; - sqlite3VdbeMemRelease(&m); - return SQLITE_OK; - - /* Jump here if database corruption is detected after m has been - ** allocated. Free the m object and return SQLITE_CORRUPT. */ -idx_rowid_corruption: - testcase( m.zMalloc!=0 ); - sqlite3VdbeMemRelease(&m); - return SQLITE_CORRUPT_BKPT; } /* -** Compare the key of the index entry that cursor pC is pointing to against -** the key string in pUnpacked. Write into *pRes a number -** that is negative, zero, or positive if pC is less than, equal to, -** or greater than pUnpacked. Return SQLITE_OK on success. -** -** pUnpacked is either created without a rowid or is truncated so that it -** omits the rowid at the end. The rowid at the end of the index entry -** is ignored as well. Hence, this routine only compares the prefixes -** of the keys prior to the final rowid, not the entire key. +** Change the value of the P2 operand for a specific instruction. +** This routine is useful for setting a jump destination. */ -SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( - VdbeCursor *pC, /* The cursor to compare against */ - UnpackedRecord *pUnpacked, /* Unpacked version of key to compare against */ - int *res /* Write the comparison result here */ -){ - i64 nCellKey = 0; - int rc; - BtCursor *pCur = pC->pCursor; - Mem m; - - assert( sqlite3BtreeCursorIsValid(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &nCellKey); - assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ - /* nCellKey will always be between 0 and 0xffffffff because of the say - ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ - if( nCellKey<=0 || nCellKey>0x7fffffff ){ - *res = 0; - return SQLITE_CORRUPT_BKPT; - } - memset(&m, 0, sizeof(m)); - rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (int)nCellKey, 1, &m); - if( rc ){ - return rc; +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + assert( p!=0 ); + assert( addr>=0 ); + if( p->nOp>addr ){ + p->aOp[addr].p2 = val; } - assert( pUnpacked->flags & UNPACKED_IGNORE_ROWID ); - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); - sqlite3VdbeMemRelease(&m); - return SQLITE_OK; } /* -** This routine sets the value to be returned by subsequent calls to -** sqlite3_changes() on the database handle 'db'. +** Change the value of the P3 operand for a specific instruction. */ -SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *db, int nChange){ - assert( sqlite3_mutex_held(db->mutex) ); - db->nChange = nChange; - db->nTotalChange += nChange; +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + assert( p!=0 ); + assert( addr>=0 ); + if( p->nOp>addr ){ + p->aOp[addr].p3 = val; + } } /* -** Set a flag in the vdbe to update the change counter when it is finalised -** or reset. +** Change the value of the P5 operand for the most recently +** added operation. */ -SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ - v->changeCntOn = 1; +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ + assert( p!=0 ); + if( p->aOp ){ + assert( p->nOp>0 ); + p->aOp[p->nOp-1].p5 = val; + } } /* -** Mark every prepared statement associated with a database connection -** as expired. -** -** An expired statement means that recompilation of the statement is -** recommend. Statements expire when things happen that make their -** programs obsolete. Removing user-defined functions or collating -** sequences, or changing an authorization function are the types of -** things that make prepared statements obsolete. +** Change the P2 operand of instruction addr so that it points to +** the address of the next instruction to be coded. */ -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db){ - Vdbe *p; - for(p = db->pVdbe; p; p=p->pNext){ - p->expired = 1; - } +SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ + assert( addr>=0 ); + sqlite3VdbeChangeP2(p, addr, p->nOp); } + /* -** Return the database associated with the Vdbe. +** If the input FuncDef structure is ephemeral, then free it. If +** the FuncDef is not ephermal, then do nothing. */ -SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe *v){ - return v->db; +static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + if( ALWAYS(pDef) && (pDef->flags & SQLITE_FUNC_EPHEM)!=0 ){ + sqlite3DbFree(db, pDef); + } } +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + /* -** Return a pointer to an sqlite3_value structure containing the value bound -** parameter iVar of VM v. Except, if the value is an SQL NULL, return -** 0 instead. Unless it is NULL, apply affinity aff (one of the SQLITE_AFF_* -** constants) to the value before returning it. -** -** The returned value must be freed by the caller using sqlite3ValueFree(). +** Delete a P4 value if necessary. */ -SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetValue(Vdbe *v, int iVar, u8 aff){ - assert( iVar>0 ); - if( v ){ - Mem *pMem = &v->aVar[iVar-1]; - if( 0==(pMem->flags & MEM_Null) ){ - sqlite3_value *pRet = sqlite3ValueNew(v->db); - if( pRet ){ - sqlite3VdbeMemCopy((Mem *)pRet, pMem); - sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); - sqlite3VdbeMemStoreType((Mem *)pRet); +static void freeP4(sqlite3 *db, int p4type, void *p4){ + if( p4 ){ + assert( db ); + switch( p4type ){ + case P4_REAL: + case P4_INT64: + case P4_DYNAMIC: + case P4_KEYINFO: + case P4_INTARRAY: + case P4_KEYINFO_HANDOFF: { + sqlite3DbFree(db, p4); + break; + } + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; + } + case P4_VDBEFUNC: { + VdbeFunc *pVdbeFunc = (VdbeFunc *)p4; + freeEphemeralFunction(db, pVdbeFunc->pFunc); + if( db->pnBytesFreed==0 ) sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); + sqlite3DbFree(db, pVdbeFunc); + break; + } + case P4_FUNCDEF: { + freeEphemeralFunction(db, (FuncDef*)p4); + break; + } + case P4_MEM: { + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + Mem *p = (Mem*)p4; + sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); + } + break; + } + case P4_VTAB : { + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); + break; } - return pRet; } } - return 0; } /* -** Configure SQL variable iVar so that binding a new value to it signals -** to sqlite3_reoptimize() that re-preparing the statement may result -** in a better query plan. +** Free the space allocated for aOp and any p4 values allocated for the +** opcodes contained within. If aOp is not NULL it is assumed to contain +** nOp entries. */ -SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ - assert( iVar>0 ); - if( iVar>32 ){ - v->expmask = 0xffffffff; - }else{ - v->expmask |= ((u32)1 << (iVar-1)); +static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ + if( aOp ){ + Op *pOp; + for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ + freeP4(db, pOp->p4type, pOp->p4.p); +#ifdef SQLITE_DEBUG + sqlite3DbFree(db, pOp->zComment); +#endif + } } + sqlite3DbFree(db, aOp); } -/************** End of vdbeaux.c *********************************************/ -/************** Begin file vdbeapi.c *****************************************/ /* -** 2004 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains code use to implement APIs that are part of the -** VDBE. +** Link the SubProgram object passed as the second argument into the linked +** list at Vdbe.pSubProgram. This list is used to delete all sub-program +** objects when the VM is no longer required. */ +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ + p->pNext = pVdbe->pProgram; + pVdbe->pProgram = p; +} -#ifndef SQLITE_OMIT_DEPRECATED /* -** Return TRUE (non-zero) of the statement supplied as an argument needs -** to be recompiled. A statement needs to be recompiled whenever the -** execution environment changes in a way that would alter the program -** that sqlite3_prepare() generates. For example, if new functions or -** collating sequences are registered or if an authorizer function is -** added or changed. +** Change N opcodes starting at addr to No-ops. */ -SQLITE_API int sqlite3_expired(sqlite3_stmt *pStmt){ - Vdbe *p = (Vdbe*)pStmt; - return p==0 || p->expired; +SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr, int N){ + if( p->aOp ){ + VdbeOp *pOp = &p->aOp[addr]; + sqlite3 *db = p->db; + while( N-- ){ + freeP4(db, pOp->p4type, pOp->p4.p); + memset(pOp, 0, sizeof(pOp[0])); + pOp->opcode = OP_Noop; + pOp++; + } + } } -#endif /* -** Check on a Vdbe to make sure it has not been finalized. Log -** an error and return true if it has been finalized (or is otherwise -** invalid). Return false if it is ok. +** Change the value of the P4 operand for a specific instruction. +** This routine is useful when a large program is loaded from a +** static array using sqlite3VdbeAddOpList but we want to make a +** few minor changes to the program. +** +** If n>=0 then the P4 operand is dynamic, meaning that a copy of +** the string is made into memory obtained from sqlite3_malloc(). +** A value of n==0 means copy bytes of zP4 up to and including the +** first null byte. If n>0 then copy n+1 bytes of zP4. +** +** If n==P4_KEYINFO it means that zP4 is a pointer to a KeyInfo structure. +** A copy is made of the KeyInfo structure into memory obtained from +** sqlite3_malloc, to be freed when the Vdbe is finalized. +** n==P4_KEYINFO_HANDOFF indicates that zP4 points to a KeyInfo structure +** stored in memory that the caller has obtained from sqlite3_malloc. The +** caller should not free the allocation, it will be freed when the Vdbe is +** finalized. +** +** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points +** to a string or structure that is guaranteed to exist for the lifetime of +** the Vdbe. In these cases we can just copy the pointer. +** +** If addr<0 then change P4 on the most recently inserted instruction. */ -static int vdbeSafety(Vdbe *p){ - if( p->db==0 ){ - sqlite3_log(SQLITE_MISUSE, "API called with finalized prepared statement"); - return 1; - }else{ - return 0; +SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ + Op *pOp; + sqlite3 *db; + assert( p!=0 ); + db = p->db; + assert( p->magic==VDBE_MAGIC_INIT ); + if( p->aOp==0 || db->mallocFailed ){ + if ( n!=P4_KEYINFO && n!=P4_VTAB ) { + freeP4(db, n, (void*)*(char**)&zP4); + } + return; } -} -static int vdbeSafetyNotNull(Vdbe *p){ - if( p==0 ){ - sqlite3_log(SQLITE_MISUSE, "API called with NULL prepared statement"); - return 1; + assert( p->nOp>0 ); + assert( addrnOp ); + if( addr<0 ){ + addr = p->nOp - 1; + } + pOp = &p->aOp[addr]; + freeP4(db, pOp->p4type, pOp->p4.p); + pOp->p4.p = 0; + if( n==P4_INT32 ){ + /* Note: this cast is safe, because the origin data point was an int + ** that was cast to a (const char *). */ + pOp->p4.i = SQLITE_PTR_TO_INT(zP4); + pOp->p4type = P4_INT32; + }else if( zP4==0 ){ + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; + }else if( n==P4_KEYINFO ){ + KeyInfo *pKeyInfo; + int nField, nByte; + + nField = ((KeyInfo*)zP4)->nField; + nByte = sizeof(*pKeyInfo) + (nField-1)*sizeof(pKeyInfo->aColl[0]) + nField; + pKeyInfo = sqlite3DbMallocRaw(0, nByte); + pOp->p4.pKeyInfo = pKeyInfo; + if( pKeyInfo ){ + u8 *aSortOrder; + memcpy((char*)pKeyInfo, zP4, nByte - nField); + aSortOrder = pKeyInfo->aSortOrder; + if( aSortOrder ){ + pKeyInfo->aSortOrder = (unsigned char*)&pKeyInfo->aColl[nField]; + memcpy(pKeyInfo->aSortOrder, aSortOrder, nField); + } + pOp->p4type = P4_KEYINFO; + }else{ + p->db->mallocFailed = 1; + pOp->p4type = P4_NOTUSED; + } + }else if( n==P4_KEYINFO_HANDOFF ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_KEYINFO; + }else if( n==P4_VTAB ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_VTAB; + sqlite3VtabLock((VTable *)zP4); + assert( ((VTable *)zP4)->db==p->db ); + }else if( n<0 ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = (signed char)n; }else{ - return vdbeSafety(p); + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; } } +#ifndef NDEBUG /* -** The following routine destroys a virtual machine that is created by -** the sqlite3_compile() routine. The integer returned is an SQLITE_ -** success/failure code that describes the result of executing the virtual -** machine. -** -** This routine sets the error code and string returned by -** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +** Change the comment on the the most recently coded instruction. Or +** insert a No-op and add the comment to that new instruction. This +** makes the code easier to read during debugging. None of this happens +** in a production build. */ -SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ - int rc; - if( pStmt==0 ){ - /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL - ** pointer is a harmless no-op. */ - rc = SQLITE_OK; - }else{ - Vdbe *v = (Vdbe*)pStmt; - sqlite3 *db = v->db; -#if SQLITE_THREADSAFE - sqlite3_mutex *mutex; -#endif - if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; -#if SQLITE_THREADSAFE - mutex = v->db->mutex; -#endif - sqlite3_mutex_enter(mutex); - rc = sqlite3VdbeFinalize(v); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(mutex); +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( !p ) return; + assert( p->nOp>0 || p->aOp==0 ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); + if( p->nOp ){ + char **pz = &p->aOp[p->nOp-1].zComment; + va_start(ap, zFormat); + sqlite3DbFree(p->db, *pz); + *pz = sqlite3VMPrintf(p->db, zFormat, ap); + va_end(ap); + } +} +SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( !p ) return; + sqlite3VdbeAddOp0(p, OP_Noop); + assert( p->nOp>0 || p->aOp==0 ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); + if( p->nOp ){ + char **pz = &p->aOp[p->nOp-1].zComment; + va_start(ap, zFormat); + sqlite3DbFree(p->db, *pz); + *pz = sqlite3VMPrintf(p->db, zFormat, ap); + va_end(ap); } - return rc; } +#endif /* NDEBUG */ /* -** Terminate the current execution of an SQL statement and reset it -** back to its starting state so that it can be reused. A success code from -** the prior execution is returned. +** Return the opcode for a given address. If the address is -1, then +** return the most recently inserted opcode. ** -** This routine sets the error code and string returned by -** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +** If a memory allocation error has occurred prior to the calling of this +** routine, then a pointer to a dummy VdbeOp will be returned. That opcode +** is readable but not writable, though it is cast to a writable value. +** The return of a dummy opcode allows the call to continue functioning +** after a OOM fault without having to check to see if the return from +** this routine is a valid pointer. But because the dummy.opcode is 0, +** dummy will never be written to. This is verified by code inspection and +** by running with Valgrind. +** +** About the #ifdef SQLITE_OMIT_TRACE: Normally, this routine is never called +** unless p->nOp>0. This is because in the absense of SQLITE_OMIT_TRACE, +** an OP_Trace instruction is always inserted by sqlite3VdbeGet() as soon as +** a new VDBE is created. So we are free to set addr to p->nOp-1 without +** having to double-check to make sure that the result is non-negative. But +** if SQLITE_OMIT_TRACE is defined, the OP_Trace is omitted and we do need to +** check the value of p->nOp-1 before continuing. */ -SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ - int rc; - if( pStmt==0 ){ - rc = SQLITE_OK; +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ + /* C89 specifies that the constant "dummy" will be initialized to all + ** zeros, which is correct. MSVC generates a warning, nevertheless. */ + static const VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ + assert( p->magic==VDBE_MAGIC_INIT ); + if( addr<0 ){ +#ifdef SQLITE_OMIT_TRACE + if( p->nOp==0 ) return (VdbeOp*)&dummy; +#endif + addr = p->nOp - 1; + } + assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); + if( p->db->mallocFailed ){ + return (VdbeOp*)&dummy; }else{ - Vdbe *v = (Vdbe*)pStmt; - sqlite3_mutex_enter(v->db->mutex); - rc = sqlite3VdbeReset(v); - sqlite3VdbeMakeReady(v, -1, 0, 0, 0, 0, 0); - assert( (rc & (v->db->errMask))==rc ); - rc = sqlite3ApiExit(v->db, rc); - sqlite3_mutex_leave(v->db->mutex); + return &p->aOp[addr]; } - return rc; } +#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) /* -** Set all the parameters in the compiled SQL statement to NULL. +** Compute a string that describes the P4 parameter for an opcode. +** Use zTemp for any required temporary buffer space. */ -SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ - int i; - int rc = SQLITE_OK; - Vdbe *p = (Vdbe*)pStmt; -#if SQLITE_THREADSAFE - sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; +static char *displayP4(Op *pOp, char *zTemp, int nTemp){ + char *zP4 = zTemp; + assert( nTemp>=20 ); + switch( pOp->p4type ){ + case P4_KEYINFO_STATIC: + case P4_KEYINFO: { + int i, j; + KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; + sqlite3_snprintf(nTemp, zTemp, "keyinfo(%d", pKeyInfo->nField); + i = sqlite3Strlen30(zTemp); + for(j=0; jnField; j++){ + CollSeq *pColl = pKeyInfo->aColl[j]; + if( pColl ){ + int n = sqlite3Strlen30(pColl->zName); + if( i+n>nTemp-6 ){ + memcpy(&zTemp[i],",...",4); + break; + } + zTemp[i++] = ','; + if( pKeyInfo->aSortOrder && pKeyInfo->aSortOrder[j] ){ + zTemp[i++] = '-'; + } + memcpy(&zTemp[i], pColl->zName,n+1); + i += n; + }else if( i+4p4.pColl; + sqlite3_snprintf(nTemp, zTemp, "collseq(%.20s)", pColl->zName); + break; + } + case P4_FUNCDEF: { + FuncDef *pDef = pOp->p4.pFunc; + sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + break; + } + case P4_INT64: { + sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); + break; + } + case P4_INT32: { + sqlite3_snprintf(nTemp, zTemp, "%d", pOp->p4.i); + break; + } + case P4_REAL: { + sqlite3_snprintf(nTemp, zTemp, "%.16g", *pOp->p4.pReal); + break; + } + case P4_MEM: { + Mem *pMem = pOp->p4.pMem; + assert( (pMem->flags & MEM_Null)==0 ); + if( pMem->flags & MEM_Str ){ + zP4 = pMem->z; + }else if( pMem->flags & MEM_Int ){ + sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); + }else if( pMem->flags & MEM_Real ){ + sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); + }else{ + assert( pMem->flags & MEM_Blob ); + zP4 = "(blob)"; + } + break; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + case P4_VTAB: { + sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; + sqlite3_snprintf(nTemp, zTemp, "vtab:%p:%p", pVtab, pVtab->pModule); + break; + } #endif - sqlite3_mutex_enter(mutex); - for(i=0; inVar; i++){ - sqlite3VdbeMemRelease(&p->aVar[i]); - p->aVar[i].flags = MEM_Null; - } - if( p->isPrepareV2 && p->expmask ){ - p->expired = 1; + case P4_INTARRAY: { + sqlite3_snprintf(nTemp, zTemp, "intarray"); + break; + } + case P4_SUBPROGRAM: { + sqlite3_snprintf(nTemp, zTemp, "program"); + break; + } + default: { + zP4 = pOp->p4.z; + if( zP4==0 ){ + zP4 = zTemp; + zTemp[0] = 0; + } + } } - sqlite3_mutex_leave(mutex); - return rc; + assert( zP4!=0 ); + return zP4; } +#endif - -/**************************** sqlite3_value_ ******************************* -** The following routines extract information from a Mem or sqlite3_value -** structure. +/* +** Declare to the Vdbe that the BTree object at db->aDb[i] is used. +** +** The prepared statements need to know in advance the complete set of +** attached databases that they will be using. A mask of these databases +** is maintained in p->btreeMask and is used for locking and other purposes. */ -SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ - Mem *p = (Mem*)pVal; - if( p->flags & (MEM_Blob|MEM_Str) ){ - sqlite3VdbeMemExpandBlob(p); - p->flags &= ~MEM_Str; - p->flags |= MEM_Blob; - return p->n ? p->z : 0; - }else{ - return sqlite3_value_text(pVal); +SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ + assert( i>=0 && idb->nDb && i<(int)sizeof(yDbMask)*8 ); + assert( i<(int)sizeof(p->btreeMask)*8 ); + p->btreeMask |= ((yDbMask)1)<db->aDb[i].pBt) ){ + p->lockMask |= ((yDbMask)1)<type; -} -/**************************** sqlite3_result_ ******************************* -** The following routines are used by user-defined functions to specify -** the function result. +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +/* +** If SQLite is compiled to support shared-cache mode and to be threadsafe, +** this routine obtains the mutex associated with each BtShared structure +** that may be accessed by the VM passed as an argument. In doing so it also +** sets the BtShared.db member of each of the BtShared structures, ensuring +** that the correct busy-handler callback is invoked if required. ** -** The setStrOrError() funtion calls sqlite3VdbeMemSetStr() to store the -** result as a string or blob but if the string or blob is too large, it -** then sets the error code to SQLITE_TOOBIG +** If SQLite is not threadsafe but does support shared-cache mode, then +** sqlite3BtreeEnter() is invoked to set the BtShared.db variables +** of all of BtShared structures accessible via the database handle +** associated with the VM. +** +** If SQLite is not threadsafe and does not support shared-cache mode, this +** function is a no-op. +** +** The p->btreeMask field is a bitmask of all btrees that the prepared +** statement p will ever use. Let N be the number of bits in p->btreeMask +** corresponding to btrees that use shared cache. Then the runtime of +** this routine is N*N. But as N is rarely more than 1, this should not +** be a problem. */ -static void setResultStrOrError( - sqlite3_context *pCtx, /* Function context */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*) /* Destructor function */ -){ - if( sqlite3VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE_TOOBIG ){ - sqlite3_result_error_toobig(pCtx); +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ + int i; + yDbMask mask; + sqlite3 *db; + Db *aDb; + int nDb; + if( p->lockMask==0 ) return; /* The common case */ + db = p->db; + aDb = db->aDb; + nDb = db->nDb; + for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + sqlite3BtreeEnter(aDb[i].pBt); + } } } -SQLITE_API void sqlite3_result_blob( - sqlite3_context *pCtx, - const void *z, - int n, - void (*xDel)(void *) -){ - assert( n>=0 ); - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, 0, xDel); -} -SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetDouble(&pCtx->s, rVal); -} -SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - pCtx->isError = SQLITE_ERROR; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - pCtx->isError = SQLITE_ERROR; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); -} #endif -SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, (i64)iVal); -} -SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, iVal); -} -SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); -} -SQLITE_API void sqlite3_result_text( - sqlite3_context *pCtx, - const char *z, - int n, - void (*xDel)(void *) -){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API void sqlite3_result_text16( - sqlite3_context *pCtx, - const void *z, - int n, - void (*xDel)(void *) -){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); -} -SQLITE_API void sqlite3_result_text16be( - sqlite3_context *pCtx, - const void *z, - int n, - void (*xDel)(void *) -){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); -} -SQLITE_API void sqlite3_result_text16le( - sqlite3_context *pCtx, - const void *z, - int n, - void (*xDel)(void *) -){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); -} -#endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemCopy(&pCtx->s, pValue); -} -SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetZeroBlob(&pCtx->s, n); -} -SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ - pCtx->isError = errCode; - if( pCtx->s.flags & MEM_Null ){ - sqlite3VdbeMemSetStr(&pCtx->s, sqlite3ErrStr(errCode), -1, - SQLITE_UTF8, SQLITE_STATIC); - } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +/* +** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). +*/ +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ + int i; + yDbMask mask; + sqlite3 *db; + Db *aDb; + int nDb; + if( p->lockMask==0 ) return; /* The common case */ + db = p->db; + aDb = db->aDb; + nDb = db->nDb; + for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + sqlite3BtreeLeave(aDb[i].pBt); + } + } } +#endif -/* Force an SQLITE_TOOBIG error. */ -SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - pCtx->isError = SQLITE_TOOBIG; - sqlite3VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, - SQLITE_UTF8, SQLITE_STATIC); +#if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +/* +** Print a single opcode. This routine is used for debugging only. +*/ +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ + char *zP4; + char zPtr[50]; + static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-4s %.2X %s\n"; + if( pOut==0 ) pOut = stdout; + zP4 = displayP4(pOp, zPtr, sizeof(zPtr)); + fprintf(pOut, zFormat1, pc, + sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, zP4, pOp->p5, +#ifdef SQLITE_DEBUG + pOp->zComment ? pOp->zComment : "" +#else + "" +#endif + ); + fflush(pOut); } +#endif -/* An SQLITE_NOMEM error. */ -SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); - pCtx->isError = SQLITE_NOMEM; - pCtx->s.db->mallocFailed = 1; +/* +** Release an array of N Mem elements +*/ +static void releaseMemArray(Mem *p, int N){ + if( p && N ){ + Mem *pEnd; + sqlite3 *db = p->db; + u8 malloc_failed = db->mallocFailed; + if( db->pnBytesFreed ){ + for(pEnd=&p[N]; pzMalloc); + } + return; + } + for(pEnd=&p[N]; pflags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ + sqlite3VdbeMemRelease(p); + }else if( p->zMalloc ){ + sqlite3DbFree(db, p->zMalloc); + p->zMalloc = 0; + } + + p->flags = MEM_Null; + } + db->mallocFailed = malloc_failed; + } } /* -** This function is called after a transaction has been committed. It -** invokes callbacks registered with sqlite3_wal_hook() as required. +** Delete a VdbeFrame object and its contents. VdbeFrame objects are +** allocated by the OP_Program opcode in sqlite3VdbeExec(). */ -static int doWalCallbacks(sqlite3 *db){ - int rc = SQLITE_OK; -#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ int i; - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); - if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ - rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); - } - } + Mem *aMem = VdbeFrameMem(p); + VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; + for(i=0; inChildCsr; i++){ + sqlite3VdbeFreeCursor(p->v, apCsr[i]); } -#endif - return rc; + releaseMemArray(aMem, p->nChildMem); + sqlite3DbFree(p->v->db, p); } +#ifndef SQLITE_OMIT_EXPLAIN /* -** Execute the statement pStmt, either until a row of data is ready, the -** statement is completely executed or an error occurs. +** Give a listing of the program in the virtual machine. ** -** This routine implements the bulk of the logic behind the sqlite_step() -** API. The only thing omitted is the automatic recompile if a -** schema change has occurred. That detail is handled by the -** outer sqlite3_step() wrapper procedure. +** The interface is the same as sqlite3VdbeExec(). But instead of +** running the code, it invokes the callback once for each instruction. +** This feature is used to implement "EXPLAIN". +** +** When p->explain==1, each instruction is listed. When +** p->explain==2, only OP_Explain instructions are listed and these +** are shown in a different format. p->explain==2 is used to implement +** EXPLAIN QUERY PLAN. +** +** When p->explain==1, first the main program is listed, then each of +** the trigger subprograms are listed one by one. */ -static int sqlite3Step(Vdbe *p){ - sqlite3 *db; - int rc; +SQLITE_PRIVATE int sqlite3VdbeList( + Vdbe *p /* The VDBE */ +){ + int nRow; /* Stop when row count reaches this */ + int nSub = 0; /* Number of sub-vdbes seen so far */ + SubProgram **apSub = 0; /* Array of sub-vdbes */ + Mem *pSub = 0; /* Memory cell hold array of subprogs */ + sqlite3 *db = p->db; /* The database connection */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + Mem *pMem = p->pResultSet = &p->aMem[1]; /* First Mem of result set */ - assert(p); - if( p->magic!=VDBE_MAGIC_RUN ){ - /* We used to require that sqlite3_reset() be called before retrying - ** sqlite3_step() after any error. But after 3.6.23, we changed this - ** so that sqlite3_reset() would be called automatically instead of - ** throwing the error. - */ - sqlite3_reset((sqlite3_stmt*)p); + assert( p->explain ); + assert( p->magic==VDBE_MAGIC_RUN ); + assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); + + /* Even though this opcode does not use dynamic strings for + ** the result, result columns may become dynamic if the user calls + ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. + */ + releaseMemArray(pMem, 8); + + if( p->rc==SQLITE_NOMEM ){ + /* This happens if a malloc() inside a call to sqlite3_column_text() or + ** sqlite3_column_text16() failed. */ + db->mallocFailed = 1; + return SQLITE_ERROR; } - /* Check that malloc() has not failed. If it has, return early. */ - db = p->db; - if( db->mallocFailed ){ - p->rc = SQLITE_NOMEM; - return SQLITE_NOMEM; + /* When the number of output rows reaches nRow, that means the + ** listing has finished and sqlite3_step() should return SQLITE_DONE. + ** nRow is the sum of the number of rows in the main program, plus + ** the sum of the number of rows in all trigger subprograms encountered + ** so far. The nRow value will increase as new trigger subprograms are + ** encountered, but p->pc will eventually catch up to nRow. + */ + nRow = p->nOp; + if( p->explain==1 ){ + /* The first 8 memory cells are used for the result set. So we will + ** commandeer the 9th cell to use as storage for an array of pointers + ** to trigger subprograms. The VDBE is guaranteed to have at least 9 + ** cells. */ + assert( p->nMem>9 ); + pSub = &p->aMem[9]; + if( pSub->flags&MEM_Blob ){ + /* On the first call to sqlite3_step(), pSub will hold a NULL. It is + ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ + nSub = pSub->n/sizeof(Vdbe*); + apSub = (SubProgram **)pSub->z; + } + for(i=0; inOp; + } } - if( p->pc<=0 && p->expired ){ - p->rc = SQLITE_SCHEMA; + do{ + i = p->pc++; + }while( iexplain==2 && p->aOp[i].opcode!=OP_Explain ); + if( i>=nRow ){ + p->rc = SQLITE_OK; + rc = SQLITE_DONE; + }else if( db->u1.isInterrupted ){ + p->rc = SQLITE_INTERRUPT; rc = SQLITE_ERROR; - goto end_of_step; - } - if( p->pc<0 ){ - /* If there are no other statements currently running, then - ** reset the interrupt flag. This prevents a call to sqlite3_interrupt - ** from interrupting a statement that has not yet started. - */ - if( db->activeVdbeCnt==0 ){ - db->u1.isInterrupted = 0; + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); + }else{ + char *z; + Op *pOp; + if( inOp ){ + /* The output line number is small enough that we are still in the + ** main program. */ + pOp = &p->aOp[i]; + }else{ + /* We are currently listing subprograms. Figure out which one and + ** pick up the appropriate opcode. */ + int j; + i -= p->nOp; + for(j=0; i>=apSub[j]->nOp; j++){ + i -= apSub[j]->nOp; + } + pOp = &apSub[j]->aOp[i]; } + if( p->explain==1 ){ + pMem->flags = MEM_Int; + pMem->type = SQLITE_INTEGER; + pMem->u.i = i; /* Program counter */ + pMem++; + + pMem->flags = MEM_Static|MEM_Str|MEM_Term; + pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30(pMem->z); + pMem->type = SQLITE_TEXT; + pMem->enc = SQLITE_UTF8; + pMem++; - assert( db->writeVdbeCnt>0 || db->autoCommit==0 || db->nDeferredCons==0 ); - -#ifndef SQLITE_OMIT_TRACE - if( db->xProfile && !db->init.busy ){ - sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + /* When an OP_Program opcode is encounter (the only opcode that has + ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms + ** kept in p->aMem[9].z to hold the new program - assuming this subprogram + ** has not already been seen. + */ + if( pOp->p4type==P4_SUBPROGRAM ){ + int nByte = (nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jp4.pProgram ) break; + } + if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, 1) ){ + apSub = (SubProgram **)pSub->z; + apSub[nSub++] = pOp->p4.pProgram; + pSub->flags |= MEM_Blob; + pSub->n = nSub*sizeof(SubProgram*); + } + } } -#endif - db->activeVdbeCnt++; - if( p->readOnly==0 ) db->writeVdbeCnt++; - p->pc = 0; - } -#ifndef SQLITE_OMIT_EXPLAIN - if( p->explain ){ - rc = sqlite3VdbeList(p); - }else -#endif /* SQLITE_OMIT_EXPLAIN */ - { - rc = sqlite3VdbeExec(p); - } + pMem->flags = MEM_Int; + pMem->u.i = pOp->p1; /* P1 */ + pMem->type = SQLITE_INTEGER; + pMem++; -#ifndef SQLITE_OMIT_TRACE - /* Invoke the profile callback if there is one - */ - if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ - sqlite3_int64 iNow; - sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); - } -#endif + pMem->flags = MEM_Int; + pMem->u.i = pOp->p2; /* P2 */ + pMem->type = SQLITE_INTEGER; + pMem++; - if( rc==SQLITE_DONE ){ - assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + pMem->flags = MEM_Int; + pMem->u.i = pOp->p3; /* P3 */ + pMem->type = SQLITE_INTEGER; + pMem++; + + if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ + assert( p->db->mallocFailed ); + return SQLITE_ERROR; } - } + pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + z = displayP4(pOp, pMem->z, 32); + if( z!=pMem->z ){ + sqlite3VdbeMemSetStr(pMem, z, -1, SQLITE_UTF8, 0); + }else{ + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30(pMem->z); + pMem->enc = SQLITE_UTF8; + } + pMem->type = SQLITE_TEXT; + pMem++; - db->errCode = rc; - if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ - p->rc = SQLITE_NOMEM; - } -end_of_step: - /* At this point local variable rc holds the value that should be - ** returned if this statement was compiled using the legacy - ** sqlite3_prepare() interface. According to the docs, this can only - ** be one of the values in the first assert() below. Variable p->rc - ** contains the value that would be returned if sqlite3_finalize() - ** were called on statement p. - */ - assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || rc==SQLITE_BUSY || rc==SQLITE_MISUSE - ); - assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); - if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - /* If this statement was prepared using sqlite3_prepare_v2(), and an - ** error has occured, then return the error code in p->rc to the - ** caller. Set the error code in the database handle to the same value. - */ - rc = db->errCode = p->rc; + if( p->explain==1 ){ + if( sqlite3VdbeMemGrow(pMem, 4, 0) ){ + assert( p->db->mallocFailed ); + return SQLITE_ERROR; + } + pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->n = 2; + sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ + pMem->type = SQLITE_TEXT; + pMem->enc = SQLITE_UTF8; + pMem++; + +#ifdef SQLITE_DEBUG + if( pOp->zComment ){ + pMem->flags = MEM_Str|MEM_Term; + pMem->z = pOp->zComment; + pMem->n = sqlite3Strlen30(pMem->z); + pMem->enc = SQLITE_UTF8; + pMem->type = SQLITE_TEXT; + }else +#endif + { + pMem->flags = MEM_Null; /* Comment */ + pMem->type = SQLITE_NULL; + } + } + + p->nResColumn = 8 - 4*(p->explain-1); + p->rc = SQLITE_OK; + rc = SQLITE_ROW; } - return (rc&db->errMask); + return rc; } +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_DEBUG /* -** This is the top-level implementation of sqlite3_step(). Call -** sqlite3Step() to do most of the work. If a schema error occurs, -** call sqlite3Reprepare() and try again. +** Print the SQL that was used to generate a VDBE program. */ -SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ - int rc = SQLITE_OK; /* Result from sqlite3Step() */ - int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ - Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ - int cnt = 0; /* Counter to prevent infinite loop of reprepares */ - sqlite3 *db; /* The database connection */ - - if( vdbeSafetyNotNull(v) ){ - return SQLITE_MISUSE_BKPT; - } - db = v->db; - sqlite3_mutex_enter(db->mutex); - while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < 5 - && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ - sqlite3_reset(pStmt); - v->expired = 0; - } - if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ - /* This case occurs after failing to recompile an sql statement. - ** The error message from the SQL compiler has already been loaded - ** into the database handle. This block copies the error message - ** from the database handle into the statement and sets the statement - ** program counter to 0 to ensure that when the statement is - ** finalized or reset the parser error message is available via - ** sqlite3_errmsg() and sqlite3_errcode(). - */ - const char *zErr = (const char *)sqlite3_value_text(db->pErr); - sqlite3DbFree(db, v->zErrMsg); - if( !db->mallocFailed ){ - v->zErrMsg = sqlite3DbStrDup(db, zErr); - v->rc = rc2; - } else { - v->zErrMsg = 0; - v->rc = rc = SQLITE_NOMEM; - } +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe *p){ + int nOp = p->nOp; + VdbeOp *pOp; + if( nOp<1 ) return; + pOp = &p->aOp[0]; + if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ + const char *z = pOp->p4.z; + while( sqlite3Isspace(*z) ) z++; + printf("SQL: [%s]\n", z); } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; } +#endif +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* -** Extract the user data from a sqlite3_context structure and return a -** pointer to it. +** Print an IOTRACE message showing SQL content. */ -SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ - assert( p && p->pFunc ); - return p->pFunc->pUserData; +SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ + int nOp = p->nOp; + VdbeOp *pOp; + if( sqlite3IoTrace==0 ) return; + if( nOp<1 ) return; + pOp = &p->aOp[0]; + if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ + int i, j; + char z[1000]; + sqlite3_snprintf(sizeof(z), z, "%s", pOp->p4.z); + for(i=0; sqlite3Isspace(z[i]); i++){} + for(j=0; z[i]; i++){ + if( sqlite3Isspace(z[i]) ){ + if( z[i-1]!=' ' ){ + z[j++] = ' '; + } + }else{ + z[j++] = z[i]; + } + } + z[j] = 0; + sqlite3IoTrace("SQL %s\n", z); + } } +#endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ /* -** Extract the user data from a sqlite3_context structure and return a -** pointer to it. +** Allocate space from a fixed size buffer and return a pointer to +** that space. If insufficient space is available, return NULL. ** -** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface -** returns a copy of the pointer to the database connection (the 1st -** parameter) of the sqlite3_create_function() and -** sqlite3_create_function16() routines that originally registered the -** application defined function. -*/ -SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pFunc ); - return p->s.db; -} - -/* -** The following is the implementation of an SQL function that always -** fails with an error message stating that the function is used in the -** wrong context. The sqlite3_overload_function() API might construct -** SQL function that use this routine so that the functions will exist -** for name resolution but are actually overloaded by the xFindFunction -** method of virtual tables. +** The pBuf parameter is the initial value of a pointer which will +** receive the new memory. pBuf is normally NULL. If pBuf is not +** NULL, it means that memory space has already been allocated and that +** this routine should not allocate any new memory. When pBuf is not +** NULL simply return pBuf. Only allocate new memory space when pBuf +** is NULL. +** +** nByte is the number of bytes of space needed. +** +** *ppFrom points to available space and pEnd points to the end of the +** available space. When space is allocated, *ppFrom is advanced past +** the end of the allocated space. +** +** *pnByte is a counter of the number of bytes of space that have failed +** to allocate. If there is insufficient space in *ppFrom to satisfy the +** request, then increment *pnByte by the amount of the request. */ -SQLITE_PRIVATE void sqlite3InvalidFunction( - sqlite3_context *context, /* The function calling context */ - int NotUsed, /* Number of arguments to the function */ - sqlite3_value **NotUsed2 /* Value of each argument */ +static void *allocSpace( + void *pBuf, /* Where return pointer will be stored */ + int nByte, /* Number of bytes to allocate */ + u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ + u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ + int *pnByte /* If allocation cannot be made, increment *pnByte */ ){ - const char *zName = context->pFunc->zName; - char *zErr; - UNUSED_PARAMETER2(NotUsed, NotUsed2); - zErr = sqlite3_mprintf( - "unable to use function %s in the requested context", zName); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); -} - -/* -** Allocate or return the aggregate context for a user function. A new -** context is allocated on the first call. Subsequent calls return the -** same context that was returned on prior calls. -*/ -SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - Mem *pMem; - assert( p && p->pFunc && p->pFunc->xStep ); - assert( sqlite3_mutex_held(p->s.db->mutex) ); - pMem = p->pMem; - testcase( nByte<0 ); - if( (pMem->flags & MEM_Agg)==0 ){ - if( nByte<=0 ){ - sqlite3VdbeMemReleaseExternal(pMem); - pMem->flags = MEM_Null; - pMem->z = 0; - }else{ - sqlite3VdbeMemGrow(pMem, nByte, 0); - pMem->flags = MEM_Agg; - pMem->u.pDef = p->pFunc; - if( pMem->z ){ - memset(pMem->z, 0, nByte); - } - } + assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); + if( pBuf ) return pBuf; + nByte = ROUND8(nByte); + if( &(*ppFrom)[nByte] <= pEnd ){ + pBuf = (void*)*ppFrom; + *ppFrom += nByte; + }else{ + *pnByte += nByte; } - return (void*)pMem->z; + return pBuf; } /* -** Return the auxilary data pointer, if any, for the iArg'th argument to -** the user-function defined by pCtx. +** Rewind the VDBE back to the beginning in preparation for +** running it. */ -SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ - VdbeFunc *pVdbeFunc; +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif + assert( p!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - pVdbeFunc = pCtx->pVdbeFunc; - if( !pVdbeFunc || iArg>=pVdbeFunc->nAux || iArg<0 ){ - return 0; + /* There should be at least one opcode. + */ + assert( p->nOp>0 ); + + /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ + p->magic = VDBE_MAGIC_RUN; + +#ifdef SQLITE_DEBUG + for(i=1; inMem; i++){ + assert( p->aMem[i].db==p->db ); } - return pVdbeFunc->apAux[iArg].pAux; +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif } /* -** Set the auxilary data pointer and delete function, for the iArg'th -** argument to the user-function defined by pCtx. Any previous value is -** deleted by calling the delete function specified when it was set. +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating stack space and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exact once on a each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, futher calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. */ -SQLITE_API void sqlite3_set_auxdata( - sqlite3_context *pCtx, - int iArg, - void *pAux, - void (*xDelete)(void*) +SQLITE_PRIVATE void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ ){ - struct AuxData *pAuxData; - VdbeFunc *pVdbeFunc; - if( iArg<0 ) goto failed; + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int n; /* Loop counter */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - pVdbeFunc = pCtx->pVdbeFunc; - if( !pVdbeFunc || pVdbeFunc->nAux<=iArg ){ - int nAux = (pVdbeFunc ? pVdbeFunc->nAux : 0); - int nMalloc = sizeof(VdbeFunc) + sizeof(struct AuxData)*iArg; - pVdbeFunc = sqlite3DbRealloc(pCtx->s.db, pVdbeFunc, nMalloc); - if( !pVdbeFunc ){ - goto failed; - } - pCtx->pVdbeFunc = pVdbeFunc; - memset(&pVdbeFunc->apAux[nAux], 0, sizeof(struct AuxData)*(iArg+1-nAux)); - pVdbeFunc->nAux = iArg+1; - pVdbeFunc->pFunc = pCtx->pFunc; - } + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + + /* For each cursor required, also allocate a memory cell. Memory + ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by + ** the vdbe program. Instead they are used to allocate space for + ** VdbeCursor/BtCursor structures. The blob of memory associated with + ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) + ** stores the blob of memory associated with cursor 1, etc. + ** + ** See also: allocateCursor(). + */ + nMem += nCursor; - pAuxData = &pVdbeFunc->apAux[iArg]; - if( pAuxData->pAux && pAuxData->xDelete ){ - pAuxData->xDelete(pAuxData->pAux); + /* Allocate space for memory registers, SQL variables, VDBE cursors and + ** an array to marshal SQL function arguments in. + */ + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[p->nOpAlloc]; /* First byte past end of zCsr[] */ + + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain && nMem<10 ){ + nMem = 10; } - pAuxData->pAux = pAux; - pAuxData->xDelete = xDelete; - return; + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; -failed: - if( xDelete ){ - xDelete(pAux); + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ + do { + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); + } + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); + + p->nCursor = (u16)nCursor; + if( p->aVar ){ + p->nVar = (ynVar)nVar; + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; + } + } + if( p->azVar ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); + } + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Null; + p->aMem[n].db = db; + } } + p->explain = pParse->explain; + sqlite3VdbeRewind(p); } -#ifndef SQLITE_OMIT_DEPRECATED /* -** Return the number of times the Step function of a aggregate has been -** called. -** -** This function is deprecated. Do not use it for new code. It is -** provide only to avoid breaking legacy code. New aggregate function -** implementations should keep their own counts within their aggregate -** context. +** Close a VDBE cursor and release all the resources that cursor +** happens to hold. */ -SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ - assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); - return p->pMem->n; -} +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ + if( pCx==0 ){ + return; + } + if( pCx->pBt ){ + sqlite3BtreeClose(pCx->pBt); + /* The pCx->pCursor will be close automatically, if it exists, by + ** the call above. */ + }else if( pCx->pCursor ){ + sqlite3BtreeCloseCursor(pCx->pCursor); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pCx->pVtabCursor ){ + sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; + const sqlite3_module *pModule = pCx->pModule; + p->inVtabMethod = 1; + pModule->xClose(pVtabCursor); + p->inVtabMethod = 0; + } #endif - -/* -** Return the number of columns in the result set for the statement pStmt. -*/ -SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ - Vdbe *pVm = (Vdbe *)pStmt; - return pVm ? pVm->nResColumn : 0; } /* -** Return the number of values available from the current row of the -** currently executing statement pStmt. +** Copy the values stored in the VdbeFrame structure to its Vdbe. This +** is used, for example, when a trigger sub-program is halted to restore +** control to the main program. */ -SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ - Vdbe *pVm = (Vdbe *)pStmt; - if( pVm==0 || pVm->pResultSet==0 ) return 0; - return pVm->nResColumn; +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ + Vdbe *v = pFrame->v; + v->aOp = pFrame->aOp; + v->nOp = pFrame->nOp; + v->aMem = pFrame->aMem; + v->nMem = pFrame->nMem; + v->apCsr = pFrame->apCsr; + v->nCursor = pFrame->nCursor; + v->db->lastRowid = pFrame->lastRowid; + v->nChange = pFrame->nChange; + return pFrame->pc; } - /* -** Check to see if column iCol of the given statement is valid. If -** it is, return a pointer to the Mem for the value of that column. -** If iCol is not valid, return a pointer to a Mem which has a value -** of NULL. +** Close all cursors. +** +** Also release any dynamic memory held by the VM in the Vdbe.aMem memory +** cell array. This is necessary as the memory cell array may contain +** pointers to VdbeFrame objects, which may in turn contain pointers to +** open cursors. */ -static Mem *columnMem(sqlite3_stmt *pStmt, int i){ - Vdbe *pVm; - int vals; - Mem *pOut; - - pVm = (Vdbe *)pStmt; - if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){ - sqlite3_mutex_enter(pVm->db->mutex); - vals = sqlite3_data_count(pStmt); - pOut = &pVm->pResultSet[i]; - }else{ - /* If the value passed as the second argument is out of range, return - ** a pointer to the following static Mem object which contains the - ** value SQL NULL. Even though the Mem structure contains an element - ** of type i64, on certain architecture (x86) with certain compiler - ** switches (-Os), gcc may align this Mem object on a 4-byte boundary - ** instead of an 8-byte one. This all works fine, except that when - ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s - ** that a Mem structure is located on an 8-byte boundary. To prevent - ** this assert() from failing, when building with SQLITE_DEBUG defined - ** using gcc, force nullMem to be 8-byte aligned using the magical - ** __attribute__((aligned(8))) macro. */ - static const Mem nullMem -#if defined(SQLITE_DEBUG) && defined(__GNUC__) - __attribute__((aligned(8))) -#endif - = {{0}, (double)0, 0, "", 0, MEM_Null, SQLITE_NULL, 0, 0, 0 }; +static void closeAllCursors(Vdbe *p){ + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + sqlite3VdbeFrameRestore(pFrame); + } + p->pFrame = 0; + p->nFrame = 0; - if( pVm && ALWAYS(pVm->db) ){ - sqlite3_mutex_enter(pVm->db->mutex); - sqlite3Error(pVm->db, SQLITE_RANGE, 0); + if( p->apCsr ){ + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursor(p, pC); + p->apCsr[i] = 0; + } } - pOut = (Mem*)&nullMem; } - return pOut; + if( p->aMem ){ + releaseMemArray(&p->aMem[1], p->nMem); + } + while( p->pDelFrame ){ + VdbeFrame *pDel = p->pDelFrame; + p->pDelFrame = pDel->pParent; + sqlite3VdbeFrameDelete(pDel); + } } /* -** This function is called after invoking an sqlite3_value_XXX function on a -** column value (i.e. a value returned by evaluating an SQL expression in the -** select list of a SELECT statement) that may cause a malloc() failure. If -** malloc() has failed, the threads mallocFailed flag is cleared and the result -** code of statement pStmt set to SQLITE_NOMEM. -** -** Specifically, this is called from within: +** Clean up the VM after execution. ** -** sqlite3_column_int() -** sqlite3_column_int64() -** sqlite3_column_text() -** sqlite3_column_text16() -** sqlite3_column_real() -** sqlite3_column_bytes() -** sqlite3_column_bytes16() -** sqiite3_column_blob() +** This routine will automatically close any cursors, lists, and/or +** sorters that were left open. It also deletes the values of +** variables in the aVar[] array. */ -static void columnMallocFailure(sqlite3_stmt *pStmt) -{ - /* If malloc() failed during an encoding conversion within an - ** sqlite3_column_XXX API, then set the return code of the statement to - ** SQLITE_NOMEM. The next call to _step() (if any) will return SQLITE_ERROR - ** and _finalize() will return NOMEM. - */ - Vdbe *p = (Vdbe *)pStmt; - if( p ){ - p->rc = sqlite3ApiExit(p->db, p->rc); - sqlite3_mutex_leave(p->db->mutex); - } +static void Cleanup(Vdbe *p){ + sqlite3 *db = p->db; + +#ifdef SQLITE_DEBUG + /* Execute assert() statements to ensure that the Vdbe.apCsr[] and + ** Vdbe.aMem[] arrays have already been cleaned up. */ + int i; + for(i=0; inCursor; i++) assert( p->apCsr==0 || p->apCsr[i]==0 ); + for(i=1; i<=p->nMem; i++) assert( p->aMem==0 || p->aMem[i].flags==MEM_Null ); +#endif + + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + p->pResultSet = 0; } -/**************************** sqlite3_column_ ******************************* -** The following routines are used to access elements of the current row -** in the result set. +/* +** Set the number of result columns that will be returned by this SQL +** statement. This is now set at compile time, rather than during +** execution of the vdbe program so that sqlite3_column_count() can +** be called on an SQL statement before sqlite3_step(). */ -SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ - const void *val; - val = sqlite3_value_blob( columnMem(pStmt,i) ); - /* Even though there is no encoding conversion, value_blob() might - ** need to call malloc() to expand the result of a zeroblob() - ** expression. - */ - columnMallocFailure(pStmt); - return val; -} -SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ - int val = sqlite3_value_bytes( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ - int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ - double val = sqlite3_value_double( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ - int val = sqlite3_value_int( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ - sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ - const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ - Mem *pOut = columnMem(pStmt, i); - if( pOut->flags&MEM_Static ){ - pOut->flags &= ~MEM_Static; - pOut->flags |= MEM_Ephem; +SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ + Mem *pColName; + int n; + sqlite3 *db = p->db; + + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + sqlite3DbFree(db, p->aColName); + n = nResColumn*COLNAME_N; + p->nResColumn = (u16)nResColumn; + p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n ); + if( p->aColName==0 ) return; + while( n-- > 0 ){ + pColName->flags = MEM_Null; + pColName->db = p->db; + pColName++; } - columnMallocFailure(pStmt); - return (sqlite3_value *)pOut; -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ - const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return val; -} -#endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ - int iType = sqlite3_value_type( columnMem(pStmt,i) ); - columnMallocFailure(pStmt); - return iType; } -/* The following function is experimental and subject to change or -** removal */ -/*int sqlite3_column_numeric_type(sqlite3_stmt *pStmt, int i){ -** return sqlite3_value_numeric_type( columnMem(pStmt,i) ); -**} -*/ - /* -** Convert the N-th element of pStmt->pColName[] into a string using -** xFunc() then return that string. If N is out of range, return 0. -** -** There are up to 5 names for each column. useType determines which -** name is returned. Here are the names: +** Set the name of the idx'th column to be returned by the SQL statement. +** zName must be a pointer to a nul terminated string. ** -** 0 The column name as it should be displayed for output -** 1 The datatype name for the column -** 2 The name of the database that the column derives from -** 3 The name of the table that the column derives from -** 4 The name of the table column that the result column derives from +** This call must be made after a call to sqlite3VdbeSetNumCols(). ** -** If the result is not a simple column reference (if it is an expression -** or a constant) then useTypes 2, 3, and 4 return NULL. +** The final parameter, xDel, must be one of SQLITE_DYNAMIC, SQLITE_STATIC +** or SQLITE_TRANSIENT. If it is SQLITE_DYNAMIC, then the buffer pointed +** to by zName will be freed by sqlite3DbFree() when the vdbe is destroyed. */ -static const void *columnName( - sqlite3_stmt *pStmt, - int N, - const void *(*xFunc)(Mem*), - int useType +SQLITE_PRIVATE int sqlite3VdbeSetColName( + Vdbe *p, /* Vdbe being configured */ + int idx, /* Index of column zName applies to */ + int var, /* One of the COLNAME_* constants */ + const char *zName, /* Pointer to buffer containing name */ + void (*xDel)(void*) /* Memory management strategy for zName */ ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; - int n; - sqlite3 *db = p->db; - - assert( db!=0 ); - n = sqlite3_column_count(pStmt); - if( N=0 ){ - N += useType*n; - sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); - ret = xFunc(&p->aColName[N]); - /* A malloc may have failed inside of the xFunc() call. If this - ** is the case, clear the mallocFailed flag and return NULL. - */ - if( db->mallocFailed ){ - db->mallocFailed = 0; - ret = 0; - } - sqlite3_mutex_leave(db->mutex); + int rc; + Mem *pColName; + assert( idxnResColumn ); + assert( vardb->mallocFailed ){ + assert( !zName || xDel!=SQLITE_DYNAMIC ); + return SQLITE_NOMEM; } - return ret; + assert( p->aColName!=0 ); + pColName = &(p->aColName[idx+var*p->nResColumn]); + rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); + assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); + return rc; } /* -** Return the name of the Nth column of the result set returned by SQL -** statement pStmt. -*/ -SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); -} -#endif - -/* -** Constraint: If you have ENABLE_COLUMN_METADATA then you must -** not define OMIT_DECLTYPE. -*/ -#if defined(SQLITE_OMIT_DECLTYPE) && defined(SQLITE_ENABLE_COLUMN_METADATA) -# error "Must not define both SQLITE_OMIT_DECLTYPE \ - and SQLITE_ENABLE_COLUMN_METADATA" -#endif - -#ifndef SQLITE_OMIT_DECLTYPE -/* -** Return the column declaration type (if applicable) of the 'i'th column -** of the result set of SQL statement pStmt. -*/ -SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); -} -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_OMIT_DECLTYPE */ - -#ifdef SQLITE_ENABLE_COLUMN_METADATA -/* -** Return the name of the database from which a result column derives. -** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. -*/ -SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); -} -#endif /* SQLITE_OMIT_UTF16 */ - -/* -** Return the name of the table from which a result column derives. -** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** A read or write transaction may or may not be active on database handle +** db. If a transaction is active, commit it. If there is a +** write-transaction spanning more than one database file, this routine +** takes care of the master journal trickery. */ -SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); -} -#endif /* SQLITE_OMIT_UTF16 */ +static int vdbeCommit(sqlite3 *db, Vdbe *p){ + int i; + int nTrans = 0; /* Number of databases with an active write-transaction */ + int rc = SQLITE_OK; + int needXcommit = 0; -/* -** Return the name of the table column from which a result column derives. -** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. -*/ -SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); -} -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_ENABLE_COLUMN_METADATA */ +#ifdef SQLITE_OMIT_VIRTUALTABLE + /* With this option, sqlite3VtabSync() is defined to be simply + ** SQLITE_OK so p is not used. + */ + UNUSED_PARAMETER(p); +#endif + /* Before doing anything else, call the xSync() callback for any + ** virtual module tables written in this transaction. This has to + ** be done before determining whether a master journal file is + ** required, as an xSync() callback may add an attached database + ** to the transaction. + */ + rc = sqlite3VtabSync(db, &p->zErrMsg); -/******************************* sqlite3_bind_ *************************** -** -** Routines used to attach values to wildcards in a compiled SQL statement. -*/ -/* -** Unbind the value bound to variable i in virtual machine p. This is the -** the same as binding a NULL value to the column. If the "i" parameter is -** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. -** -** A successful evaluation of this routine acquires the mutex on p. -** the mutex is released if any kind of error occurs. -** -** The error code stored in database p->db is overwritten with the return -** value in any case. -*/ -static int vdbeUnbind(Vdbe *p, int i){ - Mem *pVar; - if( vdbeSafetyNotNull(p) ){ - return SQLITE_MISUSE_BKPT; + /* This loop determines (a) if the commit hook should be invoked and + ** (b) how many database files have open write transactions, not + ** including the temp database. (b) is important because if more than + ** one database file has an open write transaction, a master journal + ** file is required for an atomic commit. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeIsInTrans(pBt) ){ + needXcommit = 1; + if( i!=1 ) nTrans++; + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); + } } - sqlite3_mutex_enter(p->db->mutex); - if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ - sqlite3Error(p->db, SQLITE_MISUSE, 0); - sqlite3_mutex_leave(p->db->mutex); - sqlite3_log(SQLITE_MISUSE, - "bind on a busy prepared statement: [%s]", p->zSql); - return SQLITE_MISUSE_BKPT; + if( rc!=SQLITE_OK ){ + return rc; } - if( i<1 || i>p->nVar ){ - sqlite3Error(p->db, SQLITE_RANGE, 0); - sqlite3_mutex_leave(p->db->mutex); - return SQLITE_RANGE; + + /* If there are any write-transactions at all, invoke the commit hook */ + if( needXcommit && db->xCommitCallback ){ + rc = db->xCommitCallback(db->pCommitArg); + if( rc ){ + return SQLITE_CONSTRAINT; + } } - i--; - pVar = &p->aVar[i]; - sqlite3VdbeMemRelease(pVar); - pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK, 0); - /* If the bit corresponding to this variable in Vdbe.expmask is set, then - ** binding a new value to this variable invalidates the current query plan. + /* The simple case - no more than one database file (not counting the + ** TEMP database) has a transaction active. There is no need for the + ** master-journal. ** - ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host - ** parameter in the WHERE clause might influence the choice of query plan - ** for a statement, then the statement will be automatically recompiled, - ** as if there had been a schema change, on the first sqlite3_step() call - ** following any change to the bindings of that parameter. + ** If the return value of sqlite3BtreeGetFilename() is a zero length + ** string, it means the main database is :memory: or a temp file. In + ** that case we do not support atomic multi-file commits, so use the + ** simple case then too. */ - if( p->isPrepareV2 && - ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) + if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) + || nTrans<=1 ){ - p->expired = 1; - } - return SQLITE_OK; -} - -/* -** Bind a text or BLOB value. -*/ -static int bindText( - sqlite3_stmt *pStmt, /* The statement to bind against */ - int i, /* Index of the parameter to bind */ - const void *zData, /* Pointer to the data to be bound */ - int nData, /* Number of bytes of data to be bound */ - void (*xDel)(void*), /* Destructor for the data */ - u8 encoding /* Encoding for the data */ -){ - Vdbe *p = (Vdbe *)pStmt; - Mem *pVar; - int rc; + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } + } - rc = vdbeUnbind(p, i); - if( rc==SQLITE_OK ){ - if( zData!=0 ){ - pVar = &p->aVar[i-1]; - rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); - if( rc==SQLITE_OK && encoding!=0 ){ - rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + /* Do the commit only if all databases successfully complete phase 1. + ** If one of the BtreeCommitPhaseOne() calls fails, this indicates an + ** IO error while deleting or truncating a journal file. It is unlikely, + ** but could happen. In this case abandon processing and return the error. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); } - sqlite3Error(p->db, rc, 0); - rc = sqlite3ApiExit(p->db, rc); } - sqlite3_mutex_leave(p->db->mutex); - }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ - xDel((void*)zData); + if( rc==SQLITE_OK ){ + sqlite3VtabCommit(db); + } } - return rc; -} + /* The complex case - There is a multi-file write-transaction active. + ** This requires a master journal file to ensure the transaction is + ** committed atomicly. + */ +#ifndef SQLITE_OMIT_DISKIO + else{ + sqlite3_vfs *pVfs = db->pVfs; + int needSync = 0; + char *zMaster = 0; /* File-name for the master journal */ + char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); + sqlite3_file *pMaster = 0; + i64 offset = 0; + int res; -/* -** Bind a blob value to an SQL statement variable. -*/ -SQLITE_API int sqlite3_bind_blob( - sqlite3_stmt *pStmt, - int i, - const void *zData, - int nData, - void (*xDel)(void*) -){ - return bindText(pStmt, i, zData, nData, xDel, 0); -} -SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ - int rc; - Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); - if( rc==SQLITE_OK ){ - sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); - sqlite3_mutex_leave(p->db->mutex); - } - return rc; -} -SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ - return sqlite3_bind_int64(p, i, (i64)iValue); -} -SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ - int rc; - Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); - if( rc==SQLITE_OK ){ - sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); - sqlite3_mutex_leave(p->db->mutex); - } - return rc; -} -SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ - int rc; - Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); - if( rc==SQLITE_OK ){ - sqlite3_mutex_leave(p->db->mutex); - } - return rc; -} -SQLITE_API int sqlite3_bind_text( - sqlite3_stmt *pStmt, - int i, - const char *zData, - int nData, - void (*xDel)(void*) -){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); -} -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_bind_text16( - sqlite3_stmt *pStmt, - int i, - const void *zData, - int nData, - void (*xDel)(void*) -){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); -} -#endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ - int rc; - switch( pValue->type ){ - case SQLITE_INTEGER: { - rc = sqlite3_bind_int64(pStmt, i, pValue->u.i); - break; + /* Select a master journal file name */ + do { + u32 iRandom; + sqlite3DbFree(db, zMaster); + sqlite3_randomness(sizeof(iRandom), &iRandom); + zMaster = sqlite3MPrintf(db, "%s-mj%08X", zMainFile, iRandom&0x7fffffff); + if( !zMaster ){ + return SQLITE_NOMEM; + } + sqlite3FileSuffix3(zMainFile, zMaster); + rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); + }while( rc==SQLITE_OK && res ); + if( rc==SQLITE_OK ){ + /* Open the master journal. */ + rc = sqlite3OsOpenMalloc(pVfs, zMaster, &pMaster, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_MASTER_JOURNAL, 0 + ); } - case SQLITE_FLOAT: { - rc = sqlite3_bind_double(pStmt, i, pValue->r); - break; + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, zMaster); + return rc; } - case SQLITE_BLOB: { - if( pValue->flags & MEM_Zero ){ - rc = sqlite3_bind_zeroblob(pStmt, i, pValue->u.nZero); - }else{ - rc = sqlite3_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE_TRANSIENT); + + /* Write the name of each database file in the transaction into the new + ** master journal file. If an error occurs at this point close + ** and delete the master journal file. All the individual journal files + ** still have 'null' as the master journal pointer, so they will roll + ** back independently if a failure occurs. + */ + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeIsInTrans(pBt) ){ + char const *zFile = sqlite3BtreeGetJournalname(pBt); + if( zFile==0 ){ + continue; /* Ignore TEMP and :memory: databases */ + } + assert( zFile[0]!=0 ); + if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ + needSync = 1; + } + rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); + offset += sqlite3Strlen30(zFile)+1; + if( rc!=SQLITE_OK ){ + sqlite3OsCloseFree(pMaster); + sqlite3OsDelete(pVfs, zMaster, 0); + sqlite3DbFree(db, zMaster); + return rc; + } } - break; } - case SQLITE_TEXT: { - rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE_TRANSIENT, - pValue->enc); - break; + + /* Sync the master journal file. If the IOCAP_SEQUENTIAL device + ** flag is set this is not required. + */ + if( needSync + && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) + && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) + ){ + sqlite3OsCloseFree(pMaster); + sqlite3OsDelete(pVfs, zMaster, 0); + sqlite3DbFree(db, zMaster); + return rc; } - default: { - rc = sqlite3_bind_null(pStmt, i); - break; + + /* Sync all the db files involved in the transaction. The same call + ** sets the master journal pointer in each individual journal. If + ** an error occurs here, do not delete the master journal file. + ** + ** If the error occurs during the first call to + ** sqlite3BtreeCommitPhaseOne(), then there is a chance that the + ** master journal file will be orphaned. But we cannot delete it, + ** in case the master journal file name was written into the journal + ** file before the failure occurred. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, zMaster); + } + } + sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, zMaster); + return rc; } - } - return rc; -} -SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ - int rc; - Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); - if( rc==SQLITE_OK ){ - sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); - sqlite3_mutex_leave(p->db->mutex); - } - return rc; -} -/* -** Return the number of wildcards that can be potentially bound to. -** This routine is added to support DBD::SQLite. -*/ -SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ - Vdbe *p = (Vdbe*)pStmt; - return p ? p->nVar : 0; -} + /* Delete the master journal file. This commits the transaction. After + ** doing this the directory is synced again before any individual + ** transaction files are deleted. + */ + rc = sqlite3OsDelete(pVfs, zMaster, 1); + sqlite3DbFree(db, zMaster); + zMaster = 0; + if( rc ){ + return rc; + } -/* -** Create a mapping from variable numbers to variable names -** in the Vdbe.azVar[] array, if such a mapping does not already -** exist. -*/ -static void createVarMap(Vdbe *p){ - if( !p->okVar ){ - int j; - Op *pOp; - sqlite3_mutex_enter(p->db->mutex); - /* The race condition here is harmless. If two threads call this - ** routine on the same Vdbe at the same time, they both might end - ** up initializing the Vdbe.azVar[] array. That is a little extra - ** work but it results in the same answer. + /* All files and directories have already been synced, so the following + ** calls to sqlite3BtreeCommitPhaseTwo() are only closing files and + ** deleting or truncating journals. If something goes wrong while + ** this is happening we don't really care. The integrity of the + ** transaction is already guaranteed, but some stray 'cold' journals + ** may be lying around. Returning an error code won't help matters. */ - for(j=0, pOp=p->aOp; jnOp; j++, pOp++){ - if( pOp->opcode==OP_Variable ){ - assert( pOp->p1>0 && pOp->p1<=p->nVar ); - p->azVar[pOp->p1-1] = pOp->p4.z; + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + sqlite3BtreeCommitPhaseTwo(pBt, 1); } } - p->okVar = 1; - sqlite3_mutex_leave(p->db->mutex); + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + + sqlite3VtabCommit(db); } +#endif + + return rc; } -/* -** Return the name of a wildcard parameter. Return NULL if the index -** is out of range or if the wildcard is unnamed. +/* +** This routine checks that the sqlite3.activeVdbeCnt count variable +** matches the number of vdbe's in the list sqlite3.pVdbe that are +** currently active. An assertion fails if the two counts do not match. +** This is an internal self-check only - it is not an essential processing +** step. ** -** The result is always UTF-8. +** This is a no-op if NDEBUG is defined. */ -SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ - Vdbe *p = (Vdbe*)pStmt; - if( p==0 || i<1 || i>p->nVar ){ - return 0; +#ifndef NDEBUG +static void checkActiveVdbeCnt(sqlite3 *db){ + Vdbe *p; + int cnt = 0; + int nWrite = 0; + p = db->pVdbe; + while( p ){ + if( p->magic==VDBE_MAGIC_RUN && p->pc>=0 ){ + cnt++; + if( p->readOnly==0 ) nWrite++; + } + p = p->pNext; } - createVarMap(p); - return p->azVar[i-1]; + assert( cnt==db->activeVdbeCnt ); + assert( nWrite==db->writeVdbeCnt ); } +#else +#define checkActiveVdbeCnt(x) +#endif /* -** Given a wildcard parameter name, return the index of the variable -** with that name. If there is no variable with the given name, -** return 0. +** For every Btree that in database connection db which +** has been modified, "trip" or invalidate each cursor in +** that Btree might have been modified so that the cursor +** can never be used again. This happens when a rollback +*** occurs. We have to trip all the other cursors, even +** cursor from other VMs in different database connections, +** so that none of them try to use the data at which they +** were pointing and which now may have been changed due +** to the rollback. +** +** Remember that a rollback can delete tables complete and +** reorder rootpages. So it is not sufficient just to save +** the state of the cursor. We have to invalidate the cursor +** so that it is never used again. */ -SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ +static void invalidateCursorsOnModifiedBtrees(sqlite3 *db){ int i; - if( p==0 ){ - return 0; - } - createVarMap(p); - if( zName ){ - for(i=0; inVar; i++){ - const char *z = p->azVar[i]; - if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ - return i+1; - } + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p && sqlite3BtreeIsInTrans(p) ){ + sqlite3BtreeTripAllCursors(p, SQLITE_ABORT); } } - return 0; -} -SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ - return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); } /* -** Transfer all bindings from the first statement over to the second. +** If the Vdbe passed as the first argument opened a statement-transaction, +** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or +** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement +** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the +** statement transaction is commtted. +** +** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. +** Otherwise SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ - Vdbe *pFrom = (Vdbe*)pFromStmt; - Vdbe *pTo = (Vdbe*)pToStmt; - int i; - assert( pTo->db==pFrom->db ); - assert( pTo->nVar==pFrom->nVar ); - sqlite3_mutex_enter(pTo->db->mutex); - for(i=0; inVar; i++){ - sqlite3VdbeMemMove(&pTo->aVar[i], &pFrom->aVar[i]); +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ + sqlite3 *const db = p->db; + int rc = SQLITE_OK; + + /* If p->iStatement is greater than zero, then this Vdbe opened a + ** statement transaction that should be closed here. The only exception + ** is that an IO error may have occured, causing an emergency rollback. + ** In this case (db->nStatement==0), and there is nothing to do. + */ + if( db->nStatement && p->iStatement ){ + int i; + const int iSavepoint = p->iStatement-1; + + assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); + assert( db->nStatement>0 ); + assert( p->iStatement==(db->nStatement+db->nSavepoint) ); + + for(i=0; inDb; i++){ + int rc2 = SQLITE_OK; + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc2==SQLITE_OK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + db->nStatement--; + p->iStatement = 0; + + if( rc==SQLITE_OK ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + + /* If the statement transaction is being rolled back, also restore the + ** database handles deferred constraint counter to the value it had when + ** the statement transaction was opened. */ + if( eOp==SAVEPOINT_ROLLBACK ){ + db->nDeferredCons = p->nStmtDefCons; + } } - sqlite3_mutex_leave(pTo->db->mutex); - return SQLITE_OK; + return rc; } -#ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. Internal/core SQLite code -** should call sqlite3TransferBindings. -** -** Is is misuse to call this routine with statements from different -** database connections. But as this is a deprecated interface, we -** will not bother to check for that condition. +** This function is called when a transaction opened by the database +** handle associated with the VM passed as an argument is about to be +** committed. If there are outstanding deferred foreign key constraint +** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. ** -** If the two statements contain a different number of bindings, then -** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise -** SQLITE_OK is returned. +** If there are outstanding FK violations and this function returns +** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT and write +** an error message to it. Then return SQLITE_ERROR. */ -SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ - Vdbe *pFrom = (Vdbe*)pFromStmt; - Vdbe *pTo = (Vdbe*)pToStmt; - if( pFrom->nVar!=pTo->nVar ){ +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ + sqlite3 *db = p->db; + if( (deferred && db->nDeferredCons>0) || (!deferred && p->nFkConstraint>0) ){ + p->rc = SQLITE_CONSTRAINT; + p->errorAction = OE_Abort; + sqlite3SetString(&p->zErrMsg, db, "foreign key constraint failed"); return SQLITE_ERROR; } - if( pTo->isPrepareV2 && pTo->expmask ){ - pTo->expired = 1; - } - if( pFrom->isPrepareV2 && pFrom->expmask ){ - pFrom->expired = 1; - } - return sqlite3TransferBindings(pFromStmt, pToStmt); + return SQLITE_OK; } #endif /* -** Return the sqlite3* database handle to which the prepared statement given -** in the argument belongs. This is the same database handle that was -** the first argument to the sqlite3_prepare() that was used to create -** the statement in the first place. +** This routine is called the when a VDBE tries to halt. If the VDBE +** has made changes and is in autocommit mode, then commit those +** changes. If a rollback is needed, then do the rollback. +** +** This routine is the only way to move the state of a VM from +** SQLITE_MAGIC_RUN to SQLITE_MAGIC_HALT. It is harmless to +** call this on a VM that is in the SQLITE_MAGIC_HALT state. +** +** Return an error code. If the commit could not complete because of +** lock contention, return SQLITE_BUSY. If SQLITE_BUSY is returned, it +** means the close did not happen and needs to be repeated. */ -SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ - return pStmt ? ((Vdbe*)pStmt)->db : 0; -} +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ + int rc; /* Used to store transient return codes */ + sqlite3 *db = p->db; -/* -** Return true if the prepared statement is guaranteed to not modify the -** database. -*/ -SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ - return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; -} + /* This function contains the logic that determines if a statement or + ** transaction will be committed or rolled back as a result of the + ** execution of this virtual machine. + ** + ** If any of the following errors occur: + ** + ** SQLITE_NOMEM + ** SQLITE_IOERR + ** SQLITE_FULL + ** SQLITE_INTERRUPT + ** + ** Then the internal cache might have been left in an inconsistent + ** state. We need to rollback the statement transaction, if there is + ** one, or the complete transaction if there is no statement transaction. + */ -/* -** Return a pointer to the next prepared statement after pStmt associated -** with database connection pDb. If pStmt is NULL, return the first -** prepared statement for the database connection. Return NULL if there -** are no more. -*/ -SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ - sqlite3_stmt *pNext; - sqlite3_mutex_enter(pDb->mutex); - if( pStmt==0 ){ - pNext = (sqlite3_stmt*)pDb->pVdbe; - }else{ - pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + if( p->db->mallocFailed ){ + p->rc = SQLITE_NOMEM; } - sqlite3_mutex_leave(pDb->mutex); - return pNext; + closeAllCursors(p); + if( p->magic!=VDBE_MAGIC_RUN ){ + return SQLITE_OK; + } + checkActiveVdbeCnt(db); + + /* No commit or rollback needed if the program never started */ + if( p->pc>=0 ){ + int mrc; /* Primary error code from p->rc */ + int eStatementOp = 0; + int isSpecialError; /* Set to true if a 'special' error */ + + /* Lock all btrees used by the statement */ + sqlite3VdbeEnter(p); + + /* Check for one of the special errors */ + mrc = p->rc & 0xff; + assert( p->rc!=SQLITE_IOERR_BLOCKED ); /* This error no longer exists */ + isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR + || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; + if( isSpecialError ){ + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a + ** consistent state. + ** + ** Even if the statement is read-only, it is important to perform + ** a statement or transaction rollback operation. If the error + ** occured while writing to the journal, sub-journal or database + ** file as part of an effort to free up cache space (see function + ** pagerStress() in pager.c), the rollback is required to restore + ** the pager to a consistent state. + */ + if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ + if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ + eStatementOp = SAVEPOINT_ROLLBACK; + }else{ + /* We are forced to roll back the active transaction. Before doing + ** so, abort any other statements this handle currently has active. + */ + invalidateCursorsOnModifiedBtrees(db); + sqlite3RollbackAll(db); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + } + } + } + + /* Check for immediate foreign key violations. */ + if( p->rc==SQLITE_OK ){ + sqlite3VdbeCheckFk(p, 0); + } + + /* If the auto-commit flag is set and this is the only active writer + ** VM, then we do either a commit or rollback of the current transaction. + ** + ** Note: This block also runs if one of the special errors handled + ** above has occurred. + */ + if( !sqlite3VtabInSync(db) + && db->autoCommit + && db->writeVdbeCnt==(p->readOnly==0) + ){ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ + rc = sqlite3VdbeCheckFk(p, 1); + if( rc!=SQLITE_OK ){ + if( NEVER(p->readOnly) ){ + sqlite3VdbeLeave(p); + return SQLITE_ERROR; + } + rc = SQLITE_CONSTRAINT; + }else{ + /* The auto-commit flag is true, the vdbe program was successful + ** or hit an 'OR FAIL' constraint and there are no deferred foreign + ** key constraints to hold up the transaction. This means a commit + ** is required. */ + rc = vdbeCommit(db, p); + } + if( rc==SQLITE_BUSY && p->readOnly ){ + sqlite3VdbeLeave(p); + return SQLITE_BUSY; + }else if( rc!=SQLITE_OK ){ + p->rc = rc; + sqlite3RollbackAll(db); + }else{ + db->nDeferredCons = 0; + sqlite3CommitInternalChanges(db); + } + }else{ + sqlite3RollbackAll(db); + } + db->nStatement = 0; + }else if( eStatementOp==0 ){ + if( p->rc==SQLITE_OK || p->errorAction==OE_Fail ){ + eStatementOp = SAVEPOINT_RELEASE; + }else if( p->errorAction==OE_Abort ){ + eStatementOp = SAVEPOINT_ROLLBACK; + }else{ + invalidateCursorsOnModifiedBtrees(db); + sqlite3RollbackAll(db); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + } + } + + /* If eStatementOp is non-zero, then a statement transaction needs to + ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to + ** do so. If this operation returns an error, and the current statement + ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the + ** current statement error code. + */ + if( eStatementOp ){ + rc = sqlite3VdbeCloseStatement(p, eStatementOp); + if( rc ){ + if( p->rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ){ + p->rc = rc; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + } + invalidateCursorsOnModifiedBtrees(db); + sqlite3RollbackAll(db); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + } + } + + /* If this was an INSERT, UPDATE or DELETE and no statement transaction + ** has been rolled back, update the database connection change-counter. + */ + if( p->changeCntOn ){ + if( eStatementOp!=SAVEPOINT_ROLLBACK ){ + sqlite3VdbeSetChanges(db, p->nChange); + }else{ + sqlite3VdbeSetChanges(db, 0); + } + p->nChange = 0; + } + + /* Rollback or commit any schema changes that occurred. */ + if( p->rc!=SQLITE_OK && db->flags&SQLITE_InternChanges ){ + sqlite3ResetInternalSchema(db, -1); + db->flags = (db->flags | SQLITE_InternChanges); + } + + /* Release the locks */ + sqlite3VdbeLeave(p); + } + + /* We have successfully halted and closed the VM. Record this fact. */ + if( p->pc>=0 ){ + db->activeVdbeCnt--; + if( !p->readOnly ){ + db->writeVdbeCnt--; + } + assert( db->activeVdbeCnt>=db->writeVdbeCnt ); + } + p->magic = VDBE_MAGIC_HALT; + checkActiveVdbeCnt(db); + if( p->db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + } + + /* If the auto-commit flag is set to true, then any locks that were held + ** by connection db have now been released. Call sqlite3ConnectionUnlocked() + ** to invoke any required unlock-notify callbacks. + */ + if( db->autoCommit ){ + sqlite3ConnectionUnlocked(db); + } + + assert( db->activeVdbeCnt>0 || db->autoCommit==0 || db->nStatement==0 ); + return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); } + /* -** Return the value of a status counter for a prepared statement +** Each VDBE holds the result of the most recent sqlite3_step() call +** in p->rc. This routine sets that result back to SQLITE_OK. */ -SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ - Vdbe *pVdbe = (Vdbe*)pStmt; - int v = pVdbe->aCounter[op-1]; - if( resetFlag ) pVdbe->aCounter[op-1] = 0; - return v; +SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe *p){ + p->rc = SQLITE_OK; } -/************** End of vdbeapi.c *********************************************/ -/************** Begin file vdbetrace.c ***************************************/ /* -** 2009 November 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Clean up a VDBE after execution but do not delete the VDBE just yet. +** Write any error messages into *pzErrMsg. Return the result code. ** -************************************************************************* +** After this routine is run, the VDBE should be ready to be executed +** again. ** -** This file contains code used to insert the values of host parameters -** (aka "wildcards") into the SQL text output by sqlite3_trace(). +** To look at it another way, this routine resets the state of the +** virtual machine from VDBE_MAGIC_RUN or VDBE_MAGIC_HALT back to +** VDBE_MAGIC_INIT. */ +SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ + sqlite3 *db; + db = p->db; -#ifndef SQLITE_OMIT_TRACE - -/* -** zSql is a zero-terminated string of UTF-8 SQL text. Return the number of -** bytes in this text up to but excluding the first character in -** a host parameter. If the text contains no host parameters, return -** the total number of bytes in the text. -*/ -static int findNextHostParameter(const char *zSql, int *pnToken){ - int tokenType; - int nTotal = 0; - int n; + /* If the VM did not run to completion or if it encountered an + ** error, then it might not have been halted properly. So halt + ** it now. + */ + sqlite3VdbeHalt(p); - *pnToken = 0; - while( zSql[0] ){ - n = sqlite3GetToken((u8*)zSql, &tokenType); - assert( n>0 && tokenType!=TK_ILLEGAL ); - if( tokenType==TK_VARIABLE ){ - *pnToken = n; - break; + /* If the VDBE has be run even partially, then transfer the error code + ** and error message from the VDBE into the main database structure. But + ** if the VDBE has just been set to run but has not actually executed any + ** instructions yet, leave the main database error information unchanged. + */ + if( p->pc>=0 ){ + if( p->zErrMsg ){ + sqlite3BeginBenignMalloc(); + sqlite3ValueSetStr(db->pErr,-1,p->zErrMsg,SQLITE_UTF8,SQLITE_TRANSIENT); + sqlite3EndBenignMalloc(); + db->errCode = p->rc; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + }else if( p->rc ){ + sqlite3Error(db, p->rc, 0); + }else{ + sqlite3Error(db, SQLITE_OK, 0); } - nTotal += n; - zSql += n; + if( p->runOnlyOnce ) p->expired = 1; + }else if( p->rc && p->expired ){ + /* The expired flag was set on the VDBE before the first call + ** to sqlite3_step(). For consistency (since sqlite3_step() was + ** called), set the database error in this case as well. + */ + sqlite3Error(db, p->rc, 0); + sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; } - return nTotal; -} -/* -** Return a pointer to a string in memory obtained form sqlite3DbMalloc() which -** holds a copy of zRawSql but with host parameters expanded to their -** current bindings. -** -** The calling function is responsible for making sure the memory returned -** is eventually freed. -** -** ALGORITHM: Scan the input string looking for host parameters in any of -** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within -** string literals, quoted identifier names, and comments. For text forms, -** the host parameter index is found by scanning the perpared -** statement for the corresponding OP_Variable opcode. Once the host -** parameter index is known, locate the value in p->aVar[]. Then render -** the value as a literal in place of the host parameter name. -*/ -SQLITE_PRIVATE char *sqlite3VdbeExpandSql( - Vdbe *p, /* The prepared statement being evaluated */ - const char *zRawSql /* Raw text of the SQL statement */ -){ - sqlite3 *db; /* The database connection */ - int idx = 0; /* Index of a host parameter */ - int nextIndex = 1; /* Index of next ? host parameter */ - int n; /* Length of a token prefix */ - int nToken; /* Length of the parameter token */ - int i; /* Loop counter */ - Mem *pVar; /* Value of a host parameter */ - StrAccum out; /* Accumulate the output here */ - char zBase[100]; /* Initial working space */ + /* Reclaim all memory used by the VDBE + */ + Cleanup(p); - db = p->db; - sqlite3StrAccumInit(&out, zBase, sizeof(zBase), - db->aLimit[SQLITE_LIMIT_LENGTH]); - out.db = db; - while( zRawSql[0] ){ - n = findNextHostParameter(zRawSql, &nToken); - assert( n>0 ); - sqlite3StrAccumAppend(&out, zRawSql, n); - zRawSql += n; - assert( zRawSql[0] || nToken==0 ); - if( nToken==0 ) break; - if( zRawSql[0]=='?' ){ - if( nToken>1 ){ - assert( sqlite3Isdigit(zRawSql[1]) ); - sqlite3GetInt32(&zRawSql[1], &idx); - }else{ - idx = nextIndex; - } - }else{ - assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' ); - testcase( zRawSql[0]==':' ); - testcase( zRawSql[0]=='$' ); - testcase( zRawSql[0]=='@' ); - idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); - assert( idx>0 ); - } - zRawSql += nToken; - nextIndex = idx + 1; - assert( idx>0 && idx<=p->nVar ); - pVar = &p->aVar[idx-1]; - if( pVar->flags & MEM_Null ){ - sqlite3StrAccumAppend(&out, "NULL", 4); - }else if( pVar->flags & MEM_Int ){ - sqlite3XPrintf(&out, "%lld", pVar->u.i); - }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, "%!.15g", pVar->r); - }else if( pVar->flags & MEM_Str ){ -#ifndef SQLITE_OMIT_UTF16 - u8 enc = ENC(db); - if( enc!=SQLITE_UTF8 ){ - Mem utf8; - memset(&utf8, 0, sizeof(utf8)); - utf8.db = db; - sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); - sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); - sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); - sqlite3VdbeMemRelease(&utf8); - }else -#endif - { - sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + /* Save profiling information from this VDBE run. + */ +#ifdef VDBE_PROFILE + { + FILE *out = fopen("vdbe_profile.out", "a"); + if( out ){ + int i; + fprintf(out, "---- "); + for(i=0; inOp; i++){ + fprintf(out, "%02x", p->aOp[i].opcode); } - }else if( pVar->flags & MEM_Zero ){ - sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); - }else{ - assert( pVar->flags & MEM_Blob ); - sqlite3StrAccumAppend(&out, "x'", 2); - for(i=0; in; i++){ - sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); + fprintf(out, "\n"); + for(i=0; inOp; i++){ + fprintf(out, "%6d %10lld %8lld ", + p->aOp[i].cnt, + p->aOp[i].cycles, + p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + ); + sqlite3VdbePrintOp(out, i, &p->aOp[i]); } - sqlite3StrAccumAppend(&out, "'", 1); + fclose(out); } } - return sqlite3StrAccumFinish(&out); +#endif + p->magic = VDBE_MAGIC_INIT; + return p->rc & db->errMask; +} + +/* +** Clean up and delete a VDBE after execution. Return an integer which is +** the result code. Write any error message text into *pzErrMsg. +*/ +SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ + int rc = SQLITE_OK; + if( p->magic==VDBE_MAGIC_RUN || p->magic==VDBE_MAGIC_HALT ){ + rc = sqlite3VdbeReset(p); + assert( (rc & p->db->errMask)==rc ); + } + sqlite3VdbeDelete(p); + return rc; } -#endif /* #ifndef SQLITE_OMIT_TRACE */ - -/************** End of vdbetrace.c *******************************************/ -/************** Begin file vdbe.c ********************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** The code in this file implements execution method of the -** Virtual Database Engine (VDBE). A separate file ("vdbeaux.c") -** handles housekeeping details such as creating and deleting -** VDBE instances. This file is solely interested in executing -** the VDBE program. -** -** In the external interface, an "sqlite3_stmt*" is an opaque pointer -** to a VDBE. -** -** The SQL parser generates a program which is then executed by -** the VDBE to do the work of the SQL statement. VDBE programs are -** similar in form to assembly language. The program consists of -** a linear sequence of operations. Each operation has an opcode -** and 5 operands. Operands P1, P2, and P3 are integers. Operand P4 -** is a null-terminated string. Operand P5 is an unsigned character. -** Few opcodes use all 5 operands. -** -** Computation results are stored on a set of registers numbered beginning -** with 1 and going up to Vdbe.nMem. Each register can store -** either an integer, a null-terminated string, a floating point -** number, or the SQL "NULL" value. An implicit conversion from one -** type to the other occurs as necessary. -** -** Most of the code in this file is taken up by the sqlite3VdbeExec() -** function which does the work of interpreting a VDBE program. -** But other routines are also provided to help in building up -** a program instruction by instruction. -** -** Various scripts scan this source file in order to generate HTML -** documentation, headers files, or other derived files. The formatting -** of the code in this file is, therefore, important. See other comments -** in this file for details. If in doubt, do not deviate from existing -** commenting and indentation practices when changing or adding code. +** Call the destructor for each auxdata entry in pVdbeFunc for which +** the corresponding bit in mask is clear. Auxdata entries beyond 31 +** are always destroyed. To destroy all auxdata entries, call this +** routine with mask==0. */ +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ + int i; + for(i=0; inAux; i++){ + struct AuxData *pAux = &pVdbeFunc->apAux[i]; + if( (i>31 || !(mask&(((u32)1)<pAux ){ + if( pAux->xDelete ){ + pAux->xDelete(pAux->pAux); + } + pAux->pAux = 0; + } + } +} /* -** Invoke this macro on memory cells just prior to changing the -** value of the cell. This macro verifies that shallow copies are -** not misused. +** Free all memory associated with the Vdbe passed as the second argument. +** The difference between this function and sqlite3VdbeDelete() is that +** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with +** the database connection. */ -#ifdef SQLITE_DEBUG -# define memAboutToChange(P,M) sqlite3VdbeMemPrepareToChange(P,M) -#else -# define memAboutToChange(P,M) -#endif +SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ + SubProgram *pSub, *pNext; + int i; + assert( p->db==0 || p->db==db ); + releaseMemArray(p->aVar, p->nVar); + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + for(pSub=p->pProgram; pSub; pSub=pNext){ + pNext = pSub->pNext; + vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); + sqlite3DbFree(db, pSub); + } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); + vdbeFreeOpArray(db, p->aOp, p->nOp); + sqlite3DbFree(db, p->aLabel); + sqlite3DbFree(db, p->aColName); + sqlite3DbFree(db, p->zSql); + sqlite3DbFree(db, p->pFree); + sqlite3DbFree(db, p); +} /* -** The following global variable is incremented every time a cursor -** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test -** procedures use this information to make sure that indices are -** working correctly. This variable has no function other than to -** help verify the correct operation of the library. +** Delete an entire VDBE. */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_search_count = 0; -#endif +SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ + sqlite3 *db; + + if( NEVER(p==0) ) return; + db = p->db; + if( p->pPrev ){ + p->pPrev->pNext = p->pNext; + }else{ + assert( db->pVdbe==p ); + db->pVdbe = p->pNext; + } + if( p->pNext ){ + p->pNext->pPrev = p->pPrev; + } + p->magic = VDBE_MAGIC_DEAD; + p->db = 0; + sqlite3VdbeDeleteObject(db, p); +} /* -** When this global variable is positive, it gets decremented once before -** each instruction in the VDBE. When reaches zero, the u1.isInterrupted -** field of the sqlite3 structure is set in order to simulate and interrupt. +** Make sure the cursor p is ready to read or write the row to which it +** was last positioned. Return an error code if an OOM fault or I/O error +** prevents us from positioning the cursor to its correct position. ** -** This facility is used for testing purposes only. It does not function -** in an ordinary build. +** If a MoveTo operation is pending on the given cursor, then do that +** MoveTo now. If no move is pending, check to see if the row has been +** deleted out from under the cursor and if it has, mark the row as +** a NULL row. +** +** If the cursor is already pointing to the correct row and that row has +** not been deleted out from under the cursor, then this routine is a no-op. */ +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ + if( p->deferredMoveto ){ + int res, rc; #ifdef SQLITE_TEST -SQLITE_API int sqlite3_interrupt_count = 0; + extern int sqlite3_search_count; #endif - -/* -** The next global variable is incremented each type the OP_Sort opcode -** is executed. The test procedures use this information to make sure that -** sorting is occurring or not occurring at appropriate times. This variable -** has no function other than to help verify the correct operation of the -** library. -*/ + assert( p->isTable ); + rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); + if( rc ) return rc; + p->lastRowid = p->movetoTarget; + if( res!=0 ) return SQLITE_CORRUPT_BKPT; + p->rowidIsValid = 1; #ifdef SQLITE_TEST -SQLITE_API int sqlite3_sort_count = 0; + sqlite3_search_count++; #endif - -/* -** The next global variable records the size of the largest MEM_Blob -** or MEM_Str that has been used by a VDBE opcode. The test procedures -** use this information to make sure that the zero-blob functionality -** is working correctly. This variable has no function other than to -** help verify the correct operation of the library. -*/ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_max_blobsize = 0; -static void updateMaxBlobsize(Mem *p){ - if( (p->flags & (MEM_Str|MEM_Blob))!=0 && p->n>sqlite3_max_blobsize ){ - sqlite3_max_blobsize = p->n; + p->deferredMoveto = 0; + p->cacheStatus = CACHE_STALE; + }else if( ALWAYS(p->pCursor) ){ + int hasMoved; + int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); + if( rc ) return rc; + if( hasMoved ){ + p->cacheStatus = CACHE_STALE; + p->nullRow = 1; + } } + return SQLITE_OK; } -#endif - -/* -** The next global variable is incremented each type the OP_Found opcode -** is executed. This is used to test whether or not the foreign key -** operation implemented using OP_FkIsZero is working. This variable -** has no function other than to help verify the correct operation of the -** library. -*/ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_found_count = 0; -#endif - -/* -** Test a register to see if it exceeds the current maximum blob size. -** If it does, record the new maximum blob size. -*/ -#if defined(SQLITE_TEST) && !defined(SQLITE_OMIT_BUILTIN_TEST) -# define UPDATE_MAX_BLOBSIZE(P) updateMaxBlobsize(P) -#else -# define UPDATE_MAX_BLOBSIZE(P) -#endif - -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc)) \ - { goto no_mem; } /* -** An ephemeral string value (signified by the MEM_Ephem flag) contains -** a pointer to a dynamically allocated string where some other entity -** is responsible for deallocating that string. Because the register -** does not control the string, it might be deleted without the register -** knowing it. +** The following functions: ** -** This routine converts an ephemeral string into a dynamically allocated -** string that the register itself controls. In other words, it -** converts an MEM_Ephem string into an MEM_Dyn string. -*/ -#define Deephemeralize(P) \ - if( ((P)->flags&MEM_Ephem)!=0 \ - && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} - -/* -** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*) -** P if required. +** sqlite3VdbeSerialType() +** sqlite3VdbeSerialTypeLen() +** sqlite3VdbeSerialLen() +** sqlite3VdbeSerialPut() +** sqlite3VdbeSerialGet() +** +** encapsulate the code that serializes values for storage in SQLite +** data and index records. Each serialized value consists of a +** 'serial-type' and a blob of data. The serial type is an 8-byte unsigned +** integer, stored as a varint. +** +** In an SQLite index record, the serial type is stored directly before +** the blob of data that it corresponds to. In a table record, all serial +** types are stored at the start of the record, and the blobs of data at +** the end. Hence these functions allow the caller to handle the +** serial-type and data blob seperately. +** +** The following table describes the various storage classes for data: +** +** serial type bytes of data type +** -------------- --------------- --------------- +** 0 0 NULL +** 1 1 signed integer +** 2 2 signed integer +** 3 3 signed integer +** 4 4 signed integer +** 5 6 signed integer +** 6 8 signed integer +** 7 8 IEEE float +** 8 0 Integer constant 0 +** 9 0 Integer constant 1 +** 10,11 reserved for expansion +** N>=12 and even (N-12)/2 BLOB +** N>=13 and odd (N-13)/2 text +** +** The 8 and 9 types were added in 3.3.0, file format 4. Prior versions +** of SQLite will not understand those serial types. */ -#define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) /* -** Argument pMem points at a register that will be passed to a -** user-defined function or returned to the user as the result of a query. -** This routine sets the pMem->type variable used by the sqlite3_value_*() -** routines. +** Return the serial-type for the value stored in pMem. */ -SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem){ +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ int flags = pMem->flags; - if( flags & MEM_Null ){ - pMem->type = SQLITE_NULL; - } - else if( flags & MEM_Int ){ - pMem->type = SQLITE_INTEGER; - } - else if( flags & MEM_Real ){ - pMem->type = SQLITE_FLOAT; - } - else if( flags & MEM_Str ){ - pMem->type = SQLITE_TEXT; - }else{ - pMem->type = SQLITE_BLOB; - } -} - -/* -** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL -** if we run out of memory. -*/ -static VdbeCursor *allocateCursor( - Vdbe *p, /* The virtual machine */ - int iCur, /* Index of the new VdbeCursor */ - int nField, /* Number of fields in the table or index */ - int iDb, /* When database the cursor belongs to, or -1 */ - int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ -){ - /* Find the memory cell that will be used to store the blob of memory - ** required for this VdbeCursor structure. It is convenient to use a - ** vdbe memory cell to manage the memory allocation required for a - ** VdbeCursor structure for the following reasons: - ** - ** * Sometimes cursor numbers are used for a couple of different - ** purposes in a vdbe program. The different uses might require - ** different sized allocations. Memory cells provide growable - ** allocations. - ** - ** * When using ENABLE_MEMORY_MANAGEMENT, memory cell buffers can - ** be freed lazily via the sqlite3_release_memory() API. This - ** minimizes the number of malloc calls made by the system. - ** - ** Memory cells for cursors are allocated at the top of the address - ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for - ** cursor 1 is managed by memory cell (p->nMem-1), etc. - */ - Mem *pMem = &p->aMem[p->nMem-iCur]; - - int nByte; - VdbeCursor *pCx = 0; - nByte = - ROUND8(sizeof(VdbeCursor)) + - (isBtreeCursor?sqlite3BtreeCursorSize():0) + - 2*nField*sizeof(u32); + int n; - assert( iCurnCursor ); - if( p->apCsr[iCur] ){ - sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); - p->apCsr[iCur] = 0; + if( flags&MEM_Null ){ + return 0; } - if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ - p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; - memset(pCx, 0, sizeof(VdbeCursor)); - pCx->iDb = iDb; - pCx->nField = nField; - if( nField ){ - pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))]; + if( flags&MEM_Int ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ +# define MAX_6BYTE ((((i64)0x00008000)<<32)-1) + i64 i = pMem->u.i; + u64 u; + if( file_format>=4 && (i&1)==i ){ + return 8+(u32)i; } - if( isBtreeCursor ){ - pCx->pCursor = (BtCursor*) - &pMem->z[ROUND8(sizeof(VdbeCursor))+2*nField*sizeof(u32)]; - sqlite3BtreeCursorZero(pCx->pCursor); + if( i<0 ){ + if( i<(-MAX_6BYTE) ) return 6; + /* Previous test prevents: u = -(-9223372036854775808) */ + u = -i; + }else{ + u = i; } + if( u<=127 ) return 1; + if( u<=32767 ) return 2; + if( u<=8388607 ) return 3; + if( u<=2147483647 ) return 4; + if( u<=MAX_6BYTE ) return 5; + return 6; } - return pCx; + if( flags&MEM_Real ){ + return 7; + } + assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); + n = pMem->n; + if( flags & MEM_Zero ){ + n += pMem->u.nZero; + } + assert( n>=0 ); + return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } /* -** Try to convert a value into a numeric representation if we can -** do so without loss of information. In other words, if the string -** looks like a number, convert it into a number. If it does not -** look like a number, leave it alone. +** Return the length of the data corresponding to the supplied serial-type. */ -static void applyNumericAffinity(Mem *pRec){ - if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ - double rValue; - i64 iValue; - u8 enc = pRec->enc; - if( (pRec->flags&MEM_Str)==0 ) return; - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; - pRec->flags |= MEM_Int; - }else{ - pRec->r = rValue; - pRec->flags |= MEM_Real; - } +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ + if( serial_type>=12 ){ + return (serial_type-12)/2; + }else{ + static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; + return aSize[serial_type]; } } /* -** Processing is determine by the affinity parameter: +** If we are on an architecture with mixed-endian floating +** points (ex: ARM7) then swap the lower 4 bytes with the +** upper 4 bytes. Return the result. ** -** SQLITE_AFF_INTEGER: -** SQLITE_AFF_REAL: -** SQLITE_AFF_NUMERIC: -** Try to convert pRec to an integer representation or a -** floating-point representation if an integer representation -** is not possible. Note that the integer representation is -** always preferred, even if the affinity is REAL, because -** an integer representation is more space efficient on disk. +** For most architectures, this is a no-op. ** -** SQLITE_AFF_TEXT: -** Convert pRec to a text representation. +** (later): It is reported to me that the mixed-endian problem +** on ARM7 is an issue with GCC, not with the ARM7 chip. It seems +** that early versions of GCC stored the two words of a 64-bit +** float in the wrong order. And that error has been propagated +** ever since. The blame is not necessarily with GCC, though. +** GCC might have just copying the problem from a prior compiler. +** I am also told that newer versions of GCC that follow a different +** ABI get the byte order right. ** -** SQLITE_AFF_NONE: -** No-op. pRec is unchanged. +** Developers using SQLite on an ARM7 should compile and run their +** application using -DSQLITE_DEBUG=1 at least once. With DEBUG +** enabled, some asserts below will ensure that the byte order of +** floating point values is correct. +** +** (2007-08-30) Frank van Vugt has studied this problem closely +** and has send his findings to the SQLite developers. Frank +** writes that some Linux kernels offer floating point hardware +** emulation that uses only 32-bit mantissas instead of a full +** 48-bits as required by the IEEE standard. (This is the +** CONFIG_FPE_FASTFPE option.) On such systems, floating point +** byte swapping becomes very complicated. To avoid problems, +** the necessary byte swapping is carried out using a 64-bit integer +** rather than a 64-bit float. Frank assures us that the code here +** works for him. We, the developers, have no way to independently +** verify this, but Frank seems to know what he is talking about +** so we trust him. */ -static void applyAffinity( - Mem *pRec, /* The value to apply affinity to */ - char affinity, /* The affinity to be applied */ - u8 enc /* Use this text encoding */ -){ - if( affinity==SQLITE_AFF_TEXT ){ - /* Only attempt the conversion to TEXT if there is an integer or real - ** representation (blob and NULL do not get converted) but no string - ** representation. - */ - if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc); - } - pRec->flags &= ~(MEM_Real|MEM_Int); - }else if( affinity!=SQLITE_AFF_NONE ){ - assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL - || affinity==SQLITE_AFF_NUMERIC ); - applyNumericAffinity(pRec); - if( pRec->flags & MEM_Real ){ - sqlite3VdbeIntegerAffinity(pRec); - } - } -} +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +static u64 floatSwap(u64 in){ + union { + u64 r; + u32 i[2]; + } u; + u32 t; -/* -** Try to convert the type of a function argument or a result column -** into a numeric representation. Use either INTEGER or REAL whichever -** is appropriate. But only do the conversion if it is possible without -** loss of information and return the revised type of the argument. -*/ -SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ - Mem *pMem = (Mem*)pVal; - if( pMem->type==SQLITE_TEXT ){ - applyNumericAffinity(pMem); - sqlite3VdbeMemStoreType(pMem); - } - return pMem->type; + u.r = in; + t = u.i[0]; + u.i[0] = u.i[1]; + u.i[1] = t; + return u.r; } +# define swapMixedEndianFloat(X) X = floatSwap(X) +#else +# define swapMixedEndianFloat(X) +#endif /* -** Exported version of applyAffinity(). This one works on sqlite3_value*, -** not the internal Mem* type. -*/ -SQLITE_PRIVATE void sqlite3ValueApplyAffinity( - sqlite3_value *pVal, - u8 affinity, - u8 enc -){ - applyAffinity((Mem *)pVal, affinity, enc); -} +** Write the serialized data blob for the value stored in pMem into +** buf. It is assumed that the caller has allocated sufficient space. +** Return the number of bytes written. +** +** nBuf is the amount of space left in buf[]. nBuf must always be +** large enough to hold the entire field. Except, if the field is +** a blob with a zero-filled tail, then buf[] might be just the right +** size to hold everything except for the zero-filled tail. If buf[] +** is only big enough to hold the non-zero prefix, then only write that +** prefix into buf[]. But if buf[] is large enough to hold both the +** prefix and the tail then write the prefix and set the tail to all +** zeros. +** +** Return the number of bytes actually written into buf[]. The number +** of bytes in the zero-filled tail is included in the return value only +** if those bytes were zeroed in buf[]. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, int nBuf, Mem *pMem, int file_format){ + u32 serial_type = sqlite3VdbeSerialType(pMem, file_format); + u32 len; -#ifdef SQLITE_DEBUG -/* -** Write a nice string representation of the contents of cell pMem -** into buffer zBuf, length nBuf. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ - char *zCsr = zBuf; - int f = pMem->flags; - - static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; - - if( f&MEM_Blob ){ - int i; - char c; - if( f & MEM_Dyn ){ - c = 'z'; - assert( (f & (MEM_Static|MEM_Ephem))==0 ); - }else if( f & MEM_Static ){ - c = 't'; - assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); - }else if( f & MEM_Ephem ){ - c = 'e'; - assert( (f & (MEM_Static|MEM_Dyn))==0 ); + /* Integer and Real */ + if( serial_type<=7 && serial_type>0 ){ + u64 v; + u32 i; + if( serial_type==7 ){ + assert( sizeof(v)==sizeof(pMem->r) ); + memcpy(&v, &pMem->r, sizeof(v)); + swapMixedEndianFloat(v); }else{ - c = 's'; + v = pMem->u.i; + } + len = i = sqlite3VdbeSerialTypeLen(serial_type); + assert( len<=(u32)nBuf ); + while( i-- ){ + buf[i] = (u8)(v&0xFF); + v >>= 8; } + return len; + } - sqlite3_snprintf(100, zCsr, "%c", c); - zCsr += sqlite3Strlen30(zCsr); - sqlite3_snprintf(100, zCsr, "%d[", pMem->n); - zCsr += sqlite3Strlen30(zCsr); - for(i=0; i<16 && in; i++){ - sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); - zCsr += sqlite3Strlen30(zCsr); + /* String or blob */ + if( serial_type>=12 ){ + assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) + == (int)sqlite3VdbeSerialTypeLen(serial_type) ); + assert( pMem->n<=nBuf ); + len = pMem->n; + memcpy(buf, pMem->z, len); + if( pMem->flags & MEM_Zero ){ + len += pMem->u.nZero; + assert( nBuf>=0 ); + if( len > (u32)nBuf ){ + len = (u32)nBuf; + } + memset(&buf[pMem->n], 0, len-pMem->n); } - for(i=0; i<16 && in; i++){ - char z = pMem->z[i]; - if( z<32 || z>126 ) *zCsr++ = '.'; - else *zCsr++ = z; + return len; + } + + /* NULL or constants 0 or 1 */ + return 0; +} + +/* +** Deserialize the data blob pointed to by buf as serial type serial_type +** and store the result in pMem. Return the number of bytes read. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ + switch( serial_type ){ + case 10: /* Reserved for future use */ + case 11: /* Reserved for future use */ + case 0: { /* NULL */ + pMem->flags = MEM_Null; + break; + } + case 1: { /* 1-byte signed integer */ + pMem->u.i = (signed char)buf[0]; + pMem->flags = MEM_Int; + return 1; + } + case 2: { /* 2-byte signed integer */ + pMem->u.i = (((signed char)buf[0])<<8) | buf[1]; + pMem->flags = MEM_Int; + return 2; + } + case 3: { /* 3-byte signed integer */ + pMem->u.i = (((signed char)buf[0])<<16) | (buf[1]<<8) | buf[2]; + pMem->flags = MEM_Int; + return 3; + } + case 4: { /* 4-byte signed integer */ + pMem->u.i = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + pMem->flags = MEM_Int; + return 4; + } + case 5: { /* 6-byte signed integer */ + u64 x = (((signed char)buf[0])<<8) | buf[1]; + u32 y = (buf[2]<<24) | (buf[3]<<16) | (buf[4]<<8) | buf[5]; + x = (x<<32) | y; + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + return 6; } + case 6: /* 8-byte signed integer */ + case 7: { /* IEEE floating point */ + u64 x; + u32 y; +#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) + /* Verify that integers and floating point values use the same + ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is + ** defined that 64-bit floating point values really are mixed + ** endian. + */ + static const u64 t1 = ((u64)0x3ff00000)<<32; + static const double r1 = 1.0; + u64 t2 = t1; + swapMixedEndianFloat(t2); + assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); +#endif - sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]); - zCsr += sqlite3Strlen30(zCsr); - if( f & MEM_Zero ){ - sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); - zCsr += sqlite3Strlen30(zCsr); + x = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + y = (buf[4]<<24) | (buf[5]<<16) | (buf[6]<<8) | buf[7]; + x = (x<<32) | y; + if( serial_type==6 ){ + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + }else{ + assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->r, &x, sizeof(x)); + pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; + } + return 8; } - *zCsr = '\0'; - }else if( f & MEM_Str ){ - int j, k; - zBuf[0] = ' '; - if( f & MEM_Dyn ){ - zBuf[1] = 'z'; - assert( (f & (MEM_Static|MEM_Ephem))==0 ); - }else if( f & MEM_Static ){ - zBuf[1] = 't'; - assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); - }else if( f & MEM_Ephem ){ - zBuf[1] = 'e'; - assert( (f & (MEM_Static|MEM_Dyn))==0 ); - }else{ - zBuf[1] = 's'; + case 8: /* Integer 0 */ + case 9: { /* Integer 1 */ + pMem->u.i = serial_type-8; + pMem->flags = MEM_Int; + return 0; } - k = 2; - sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); - k += sqlite3Strlen30(&zBuf[k]); - zBuf[k++] = '['; - for(j=0; j<15 && jn; j++){ - u8 c = pMem->z[j]; - if( c>=0x20 && c<0x7f ){ - zBuf[k++] = c; + default: { + u32 len = (serial_type-12)/2; + pMem->z = (char *)buf; + pMem->n = len; + pMem->xDel = 0; + if( serial_type&0x01 ){ + pMem->flags = MEM_Str | MEM_Ephem; }else{ - zBuf[k++] = '.'; + pMem->flags = MEM_Blob | MEM_Ephem; } + return len; } - zBuf[k++] = ']'; - sqlite3_snprintf(100,&zBuf[k], encnames[pMem->enc]); - k += sqlite3Strlen30(&zBuf[k]); - zBuf[k++] = 0; } + return 0; } -#endif -#ifdef SQLITE_DEBUG + /* -** Print the value of a register for tracing purposes: -*/ -static void memTracePrint(FILE *out, Mem *p){ - if( p->flags & MEM_Null ){ - fprintf(out, " NULL"); - }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ - fprintf(out, " si:%lld", p->u.i); - }else if( p->flags & MEM_Int ){ - fprintf(out, " i:%lld", p->u.i); -#ifndef SQLITE_OMIT_FLOATING_POINT - }else if( p->flags & MEM_Real ){ - fprintf(out, " r:%g", p->r); -#endif - }else if( p->flags & MEM_RowSet ){ - fprintf(out, " (rowset)"); +** Given the nKey-byte encoding of a record in pKey[], parse the +** record into a UnpackedRecord structure. Return a pointer to +** that structure. +** +** The calling function might provide szSpace bytes of memory +** space at pSpace. This space can be used to hold the returned +** VDbeParsedRecord structure if it is large enough. If it is +** not big enough, space is obtained from sqlite3_malloc(). +** +** The returned structure should be closed by a call to +** sqlite3VdbeDeleteUnpackedRecord(). +*/ +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack( + KeyInfo *pKeyInfo, /* Information about the record format */ + int nKey, /* Size of the binary record */ + const void *pKey, /* The binary record */ + char *pSpace, /* Unaligned space available to hold the object */ + int szSpace /* Size of pSpace[] in bytes */ +){ + const unsigned char *aKey = (const unsigned char *)pKey; + UnpackedRecord *p; /* The unpacked record that we will return */ + int nByte; /* Memory space needed to hold p, in bytes */ + int d; + u32 idx; + u16 u; /* Unsigned loop counter */ + u32 szHdr; + Mem *pMem; + int nOff; /* Increase pSpace by this much to 8-byte align it */ + + /* + ** We want to shift the pointer pSpace up such that it is 8-byte aligned. + ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift + ** it by. If pSpace is already 8-byte aligned, nOff should be zero. + */ + nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; + pSpace += nOff; + szSpace -= nOff; + nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); + if( nByte>szSpace ){ + p = sqlite3DbMallocRaw(pKeyInfo->db, nByte); + if( p==0 ) return 0; + p->flags = UNPACKED_NEED_FREE | UNPACKED_NEED_DESTROY; }else{ - char zBuf[200]; - sqlite3VdbeMemPrettyPrint(p, zBuf); - fprintf(out, " "); - fprintf(out, "%s", zBuf); + p = (UnpackedRecord*)pSpace; + p->flags = UNPACKED_NEED_DESTROY; } + p->pKeyInfo = pKeyInfo; + p->nField = pKeyInfo->nField + 1; + p->aMem = pMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + idx = getVarint32(aKey, szHdr); + d = szHdr; + u = 0; + while( idxnField && d<=nKey ){ + u32 serial_type; + + idx += getVarint32(&aKey[idx], serial_type); + pMem->enc = pKeyInfo->enc; + pMem->db = pKeyInfo->db; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ + pMem->zMalloc = 0; + d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + pMem++; + u++; + } + assert( u<=pKeyInfo->nField + 1 ); + p->nField = u; + return (void*)p; } -static void registerTrace(FILE *out, int iReg, Mem *p){ - fprintf(out, "REG[%d] = ", iReg); - memTracePrint(out, p); - fprintf(out, "\n"); -} -#endif +/* +** This routine destroys a UnpackedRecord object. +*/ +SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ #ifdef SQLITE_DEBUG -# define REGISTER_TRACE(R,M) if(p->trace)registerTrace(p->trace,R,M) -#else -# define REGISTER_TRACE(R,M) -#endif - + int i; + Mem *pMem; -#ifdef VDBE_PROFILE + assert( p!=0 ); + assert( p->flags & UNPACKED_NEED_DESTROY ); + for(i=0, pMem=p->aMem; inField; i++, pMem++){ + /* The unpacked record is always constructed by the + ** sqlite3VdbeUnpackRecord() function above, which makes all + ** strings and blobs static. And none of the elements are + ** ever transformed, so there is never anything to delete. + */ + if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); + } +#endif + if( p->flags & UNPACKED_NEED_FREE ){ + sqlite3DbFree(p->pKeyInfo->db, p); + } +} -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of vdbe.c *********************/ -/************** Begin file hwtime.h ******************************************/ /* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This function compares the two table rows or index records +** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero +** or positive integer if key1 is less than, equal to or +** greater than key2. The {nKey1, pKey1} key must be a blob +** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** key must be a parsed key such as obtained from +** sqlite3VdbeParseRecord. ** -****************************************************************************** +** Key1 and Key2 do not have to contain the same number of fields. +** The key with fewer fields is usually compares less than the +** longer key. However if the UNPACKED_INCRKEY flags in pPKey2 is set +** and the common prefixes are equal, then key1 is less than key2. +** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are +** equal, then the keys are considered to be equal and +** the parts beyond the common prefix are ignored. ** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. +** If the UNPACKED_IGNORE_ROWID flag is set, then the last byte of +** the header of pKey1 is ignored. It is assumed that pKey1 is +** an index key, and thus ends with a rowid value. The last byte +** of the header will therefore be the serial type of the rowid: +** one of 1, 2, 3, 4, 5, 6, 8, or 9 - the integer serial types. +** The serial type of the final rowid will always be a single byte. +** By ignoring this last byte of the header, we force the comparison +** to ignore the rowid at the end of key1. */ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } +SQLITE_PRIVATE int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + int d1; /* Offset into aKey[] of next data element */ + u32 idx1; /* Offset into aKey[] of next header element */ + u32 szHdr1; /* Number of bytes in header */ + int i = 0; + int nField; + int rc = 0; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + KeyInfo *pKeyInfo; + Mem mem1; - #elif defined(_MSC_VER) + pKeyInfo = pPKey2->pKeyInfo; + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } + /* Compilers may complain that mem1.u.i is potentially uninitialized. + ** We could initialize it, as shown here, to silence those complaints. + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing + ** the unnecessary initialization has a measurable negative performance + ** impact, since this routine is a very high runner. And so, we choose + ** to ignore the compiler warnings and leave this variable uninitialized. + */ + /* mem1.u.i = 0; // not needed, here to silence compiler warning */ + + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){ + szHdr1--; } + nField = pKeyInfo->nField; + while( idx1nField ){ + u32 serial_type1; - #endif + /* Read the serial types for the next element in each key. */ + idx1 += getVarint32( aKey1+idx1, serial_type1 ); + if( d1>=nKey1 && sqlite3VdbeSerialTypeLen(serial_type1)>0 ) break; -#elif (defined(__GNUC__) && defined(__x86_64__)) + /* Extract the values to be compared. + */ + d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif (defined(__GNUC__) && defined(__ppc__)) + /* Do the comparison + */ + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], + iaColl[i] : 0); + if( rc!=0 ){ + assert( mem1.zMalloc==0 ); /* See comment below */ - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; + /* Invert the result if we are using DESC sort order. */ + if( pKeyInfo->aSortOrder && iaSortOrder[i] ){ + rc = -rc; + } + + /* If the PREFIX_SEARCH flag is set and all fields except the final + ** rowid field were equal, then clear the PREFIX_SEARCH flag and set + ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1). + ** This is used by the OP_IsUnique opcode. + */ + if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){ + assert( idx1==szHdr1 && rc ); + assert( mem1.flags & MEM_Int ); + pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH; + pPKey2->rowid = mem1.u.i; + } + + return rc; + } + i++; } -#else - - #error Need implementation of sqlite3Hwtime() for your platform. - - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(_HWTIME_H_) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in vdbe.c ***********************/ + assert( mem1.zMalloc==0 ); -#endif + /* rc==0 here means that one of the keys ran out of fields and + ** all the fields up to that point were equal. If the UNPACKED_INCRKEY + ** flag is set, then break the tie by treating key2 as larger. + ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes + ** are considered to be equal. Otherwise, the longer key is the + ** larger. As it happens, the pPKey2 will always be the longer + ** if there is a difference. + */ + assert( rc==0 ); + if( pPKey2->flags & UNPACKED_INCRKEY ){ + rc = -1; + }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){ + /* Leave rc==0 */ + }else if( idx1u1.isInterrupted ) goto abort_due_to_interrupt; +** pCur might be pointing to text obtained from a corrupt database file. +** So the content cannot be trusted. Do appropriate checks on the content. +*/ +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ + i64 nCellKey = 0; + int rc; + u32 szHdr; /* Size of the header */ + u32 typeRowid; /* Serial type of the rowid */ + u32 lenRowid; /* Size of the rowid */ + Mem m, v; + UNUSED_PARAMETER(db); + + /* Get the size of the index entry. Only indices entries of less + ** than 2GiB are support - anything large must be database corruption. + ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so + ** this code can safely assume that nCellKey is 32-bits + */ + assert( sqlite3BtreeCursorIsValid(pCur) ); + rc = sqlite3BtreeKeySize(pCur, &nCellKey); + assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); + + /* Read in the complete content of the index entry */ + memset(&m, 0, sizeof(m)); + rc = sqlite3VdbeMemFromBtree(pCur, 0, (int)nCellKey, 1, &m); + if( rc ){ + return rc; + } + + /* The index entry must begin with a header size */ + (void)getVarint32((u8*)m.z, szHdr); + testcase( szHdr==3 ); + testcase( szHdr==m.n ); + if( unlikely(szHdr<3 || (int)szHdr>m.n) ){ + goto idx_rowid_corruption; + } + + /* The last field of the index should be an integer - the ROWID. + ** Verify that the last entry really is an integer. */ + (void)getVarint32((u8*)&m.z[szHdr-1], typeRowid); + testcase( typeRowid==1 ); + testcase( typeRowid==2 ); + testcase( typeRowid==3 ); + testcase( typeRowid==4 ); + testcase( typeRowid==5 ); + testcase( typeRowid==6 ); + testcase( typeRowid==8 ); + testcase( typeRowid==9 ); + if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ + goto idx_rowid_corruption; + } + lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); + testcase( (u32)m.n==szHdr+lenRowid ); + if( unlikely((u32)m.npSavepoint; p; p=p->pNext) n++; - assert( n==(db->nSavepoint + db->isTransactionSavepoint) ); - return 1; +SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( + VdbeCursor *pC, /* The cursor to compare against */ + UnpackedRecord *pUnpacked, /* Unpacked version of key to compare against */ + int *res /* Write the comparison result here */ +){ + i64 nCellKey = 0; + int rc; + BtCursor *pCur = pC->pCursor; + Mem m; + + assert( sqlite3BtreeCursorIsValid(pCur) ); + rc = sqlite3BtreeKeySize(pCur, &nCellKey); + assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + /* nCellKey will always be between 0 and 0xffffffff because of the say + ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ + if( nCellKey<=0 || nCellKey>0x7fffffff ){ + *res = 0; + return SQLITE_CORRUPT_BKPT; + } + memset(&m, 0, sizeof(m)); + rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (int)nCellKey, 1, &m); + if( rc ){ + return rc; + } + assert( pUnpacked->flags & UNPACKED_IGNORE_ROWID ); + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); + sqlite3VdbeMemRelease(&m); + return SQLITE_OK; } -#endif /* -** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored -** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored -** in memory obtained from sqlite3DbMalloc). +** This routine sets the value to be returned by subsequent calls to +** sqlite3_changes() on the database handle 'db'. */ -static void importVtabErrMsg(Vdbe *p, sqlite3_vtab *pVtab){ - sqlite3 *db = p->db; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *db, int nChange){ + assert( sqlite3_mutex_held(db->mutex) ); + db->nChange = nChange; + db->nTotalChange += nChange; } +/* +** Set a flag in the vdbe to update the change counter when it is finalised +** or reset. +*/ +SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ + v->changeCntOn = 1; +} /* -** Execute as much of a VDBE program as we can then return. +** Mark every prepared statement associated with a database connection +** as expired. ** -** sqlite3VdbeMakeReady() must be called before this routine in order to -** close the program with a final OP_Halt and to set up the callbacks -** and the error message pointer. +** An expired statement means that recompilation of the statement is +** recommend. Statements expire when things happen that make their +** programs obsolete. Removing user-defined functions or collating +** sequences, or changing an authorization function are the types of +** things that make prepared statements obsolete. +*/ +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db){ + Vdbe *p; + for(p = db->pVdbe; p; p=p->pNext){ + p->expired = 1; + } +} + +/* +** Return the database associated with the Vdbe. +*/ +SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe *v){ + return v->db; +} + +/* +** Return a pointer to an sqlite3_value structure containing the value bound +** parameter iVar of VM v. Except, if the value is an SQL NULL, return +** 0 instead. Unless it is NULL, apply affinity aff (one of the SQLITE_AFF_* +** constants) to the value before returning it. ** -** Whenever a row or result data is available, this routine will either -** invoke the result callback (if there is one) or return with -** SQLITE_ROW. +** The returned value must be freed by the caller using sqlite3ValueFree(). +*/ +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetValue(Vdbe *v, int iVar, u8 aff){ + assert( iVar>0 ); + if( v ){ + Mem *pMem = &v->aVar[iVar-1]; + if( 0==(pMem->flags & MEM_Null) ){ + sqlite3_value *pRet = sqlite3ValueNew(v->db); + if( pRet ){ + sqlite3VdbeMemCopy((Mem *)pRet, pMem); + sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); + sqlite3VdbeMemStoreType((Mem *)pRet); + } + return pRet; + } + } + return 0; +} + +/* +** Configure SQL variable iVar so that binding a new value to it signals +** to sqlite3_reoptimize() that re-preparing the statement may result +** in a better query plan. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ + assert( iVar>0 ); + if( iVar>32 ){ + v->expmask = 0xffffffff; + }else{ + v->expmask |= ((u32)1 << (iVar-1)); + } +} + +/************** End of vdbeaux.c *********************************************/ +/************** Begin file vdbeapi.c *****************************************/ +/* +** 2004 May 26 ** -** If an attempt is made to open a locked database, then this routine -** will either invoke the busy callback (if there is one) or it will -** return SQLITE_BUSY. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** If an error occurs, an error message is written to memory obtained -** from sqlite3_malloc() and p->zErrMsg is made to point to that memory. -** The error code is stored in p->rc and this routine returns SQLITE_ERROR. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** If the callback ever returns non-zero, then the program exits -** immediately. There will be no error message but the p->rc field is -** set to SQLITE_ABORT and this routine will return SQLITE_ERROR. +************************************************************************* ** -** A memory allocation error causes p->rc to be set to SQLITE_NOMEM and this -** routine to return SQLITE_ERROR. +** This file contains code use to implement APIs that are part of the +** VDBE. +*/ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Return TRUE (non-zero) of the statement supplied as an argument needs +** to be recompiled. A statement needs to be recompiled whenever the +** execution environment changes in a way that would alter the program +** that sqlite3_prepare() generates. For example, if new functions or +** collating sequences are registered or if an authorizer function is +** added or changed. +*/ +SQLITE_API int sqlite3_expired(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + return p==0 || p->expired; +} +#endif + +/* +** Check on a Vdbe to make sure it has not been finalized. Log +** an error and return true if it has been finalized (or is otherwise +** invalid). Return false if it is ok. +*/ +static int vdbeSafety(Vdbe *p){ + if( p->db==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with finalized prepared statement"); + return 1; + }else{ + return 0; + } +} +static int vdbeSafetyNotNull(Vdbe *p){ + if( p==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with NULL prepared statement"); + return 1; + }else{ + return vdbeSafety(p); + } +} + +/* +** The following routine destroys a virtual machine that is created by +** the sqlite3_compile() routine. The integer returned is an SQLITE_ +** success/failure code that describes the result of executing the virtual +** machine. ** -** Other fatal errors return SQLITE_ERROR. +** This routine sets the error code and string returned by +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ + int rc; + if( pStmt==0 ){ + /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL + ** pointer is a harmless no-op. */ + rc = SQLITE_OK; + }else{ + Vdbe *v = (Vdbe*)pStmt; + sqlite3 *db = v->db; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; +#endif + if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; +#if SQLITE_THREADSAFE + mutex = v->db->mutex; +#endif + sqlite3_mutex_enter(mutex); + rc = sqlite3VdbeFinalize(v); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(mutex); + } + return rc; +} + +/* +** Terminate the current execution of an SQL statement and reset it +** back to its starting state so that it can be reused. A success code from +** the prior execution is returned. ** -** After this routine has finished, sqlite3VdbeFinalize() should be -** used to clean up the mess that was left behind. +** This routine sets the error code and string returned by +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_PRIVATE int sqlite3VdbeExec( - Vdbe *p /* The VDBE */ -){ - int pc=0; /* The program counter */ - Op *aOp = p->aOp; /* Copy of p->aOp */ - Op *pOp; /* Current operation */ - int rc = SQLITE_OK; /* Value to return */ - sqlite3 *db = p->db; /* The database */ - u8 resetSchemaOnFault = 0; /* Reset schema after an error if true */ - u8 encoding = ENC(db); /* The database encoding */ -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - int checkProgress; /* True if progress callbacks are enabled */ - int nProgressOps = 0; /* Opcodes executed since progress callback. */ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ + int rc; + if( pStmt==0 ){ + rc = SQLITE_OK; + }else{ + Vdbe *v = (Vdbe*)pStmt; + sqlite3_mutex_enter(v->db->mutex); + rc = sqlite3VdbeReset(v); + sqlite3VdbeRewind(v); + assert( (rc & (v->db->errMask))==rc ); + rc = sqlite3ApiExit(v->db, rc); + sqlite3_mutex_leave(v->db->mutex); + } + return rc; +} + +/* +** Set all the parameters in the compiled SQL statement to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ + int i; + int rc = SQLITE_OK; + Vdbe *p = (Vdbe*)pStmt; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; #endif - Mem *aMem = p->aMem; /* Copy of p->aMem */ - Mem *pIn1 = 0; /* 1st input operand */ - Mem *pIn2 = 0; /* 2nd input operand */ - Mem *pIn3 = 0; /* 3rd input operand */ - Mem *pOut = 0; /* Output operand */ - int iCompare = 0; /* Result of last OP_Compare operation */ - int *aPermute = 0; /* Permutation of columns for OP_Compare */ -#ifdef VDBE_PROFILE - u64 start; /* CPU clock count at start of opcode */ - int origPc; /* Program counter at start of opcode */ + sqlite3_mutex_enter(mutex); + for(i=0; inVar; i++){ + sqlite3VdbeMemRelease(&p->aVar[i]); + p->aVar[i].flags = MEM_Null; + } + if( p->isPrepareV2 && p->expmask ){ + p->expired = 1; + } + sqlite3_mutex_leave(mutex); + return rc; +} + + +/**************************** sqlite3_value_ ******************************* +** The following routines extract information from a Mem or sqlite3_value +** structure. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ + Mem *p = (Mem*)pVal; + if( p->flags & (MEM_Blob|MEM_Str) ){ + sqlite3VdbeMemExpandBlob(p); + p->flags &= ~MEM_Str; + p->flags |= MEM_Blob; + return p->n ? p->z : 0; + }else{ + return sqlite3_value_text(pVal); + } +} +SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ + return sqlite3ValueBytes(pVal, SQLITE_UTF8); +} +SQLITE_API int sqlite3_value_bytes16(sqlite3_value *pVal){ + return sqlite3ValueBytes(pVal, SQLITE_UTF16NATIVE); +} +SQLITE_API double sqlite3_value_double(sqlite3_value *pVal){ + return sqlite3VdbeRealValue((Mem*)pVal); +} +SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ + return (int)sqlite3VdbeIntValue((Mem*)pVal); +} +SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ + return sqlite3VdbeIntValue((Mem*)pVal); +} +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ + return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_value_text16(sqlite3_value* pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16NATIVE); +} +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value *pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16BE); +} +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16LE); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ + return pVal->type; +} + +/**************************** sqlite3_result_ ******************************* +** The following routines are used by user-defined functions to specify +** the function result. +** +** The setStrOrError() funtion calls sqlite3VdbeMemSetStr() to store the +** result as a string or blob but if the string or blob is too large, it +** then sets the error code to SQLITE_TOOBIG +*/ +static void setResultStrOrError( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ +){ + if( sqlite3VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(pCtx); + } +} +SQLITE_API void sqlite3_result_blob( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( n>=0 ); + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, 0, xDel); +} +SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetDouble(&pCtx->s, rVal); +} +SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pCtx->isError = SQLITE_ERROR; + sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pCtx->isError = SQLITE_ERROR; + sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); +} #endif - /******************************************************************** - ** Automatically generated code - ** - ** The following union is automatically generated by the - ** vdbe-compress.tcl script. The purpose of this union is to - ** reduce the amount of stack space required by this function. - ** See comments in the vdbe-compress.tcl script for details. - */ - union vdbeExecUnion { - struct OP_Yield_stack_vars { - int pcDest; - } aa; - struct OP_Variable_stack_vars { - Mem *pVar; /* Value being transferred */ - } ab; - struct OP_Move_stack_vars { - char *zMalloc; /* Holding variable for allocated memory */ - int n; /* Number of registers left to copy */ - int p1; /* Register to copy from */ - int p2; /* Register to copy to */ - } ac; - struct OP_ResultRow_stack_vars { - Mem *pMem; - int i; - } ad; - struct OP_Concat_stack_vars { - i64 nByte; - } ae; - struct OP_Remainder_stack_vars { - int flags; /* Combined MEM_* flags from both inputs */ - i64 iA; /* Integer value of left operand */ - i64 iB; /* Integer value of right operand */ - double rA; /* Real value of left operand */ - double rB; /* Real value of right operand */ - } af; - struct OP_Function_stack_vars { - int i; - Mem *pArg; - sqlite3_context ctx; - sqlite3_value **apVal; - int n; - } ag; - struct OP_ShiftRight_stack_vars { - i64 a; - i64 b; - } ah; - struct OP_Ge_stack_vars { - int res; /* Result of the comparison of pIn1 against pIn3 */ - char affinity; /* Affinity to use for comparison */ - u16 flags1; /* Copy of initial value of pIn1->flags */ - u16 flags3; /* Copy of initial value of pIn3->flags */ - } ai; - struct OP_Compare_stack_vars { - int n; - int i; - int p1; - int p2; - const KeyInfo *pKeyInfo; - int idx; - CollSeq *pColl; /* Collating sequence to use on this term */ - int bRev; /* True for DESCENDING sort order */ - } aj; - struct OP_Or_stack_vars { - int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ - int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ - } ak; - struct OP_IfNot_stack_vars { - int c; - } al; - struct OP_Column_stack_vars { - u32 payloadSize; /* Number of bytes in the record */ - i64 payloadSize64; /* Number of bytes in the record */ - int p1; /* P1 value of the opcode */ - int p2; /* column number to retrieve */ - VdbeCursor *pC; /* The VDBE cursor */ - char *zRec; /* Pointer to complete record-data */ - BtCursor *pCrsr; /* The BTree cursor */ - u32 *aType; /* aType[i] holds the numeric type of the i-th column */ - u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ - int nField; /* number of fields in the record */ - int len; /* The length of the serialized data for the column */ - int i; /* Loop counter */ - char *zData; /* Part of the record being decoded */ - Mem *pDest; /* Where to write the extracted value */ - Mem sMem; /* For storing the record being decoded */ - u8 *zIdx; /* Index into header */ - u8 *zEndHdr; /* Pointer to first byte after the header */ - u32 offset; /* Offset into the data */ - u32 szField; /* Number of bytes in the content of a field */ - int szHdr; /* Size of the header size field at start of record */ - int avail; /* Number of bytes of available data */ - Mem *pReg; /* PseudoTable input register */ - } am; - struct OP_Affinity_stack_vars { - const char *zAffinity; /* The affinity to be applied */ - char cAff; /* A single character of affinity */ - } an; - struct OP_MakeRecord_stack_vars { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ - Mem *pRec; /* The new record */ - u64 nData; /* Number of bytes of data space */ - int nHdr; /* Number of bytes of header space */ - i64 nByte; /* Data space required for this record */ - int nZero; /* Number of zero bytes at the end of the record */ - int nVarint; /* Number of bytes in a varint */ - u32 serial_type; /* Type field */ - Mem *pData0; /* First field to be combined into the record */ - Mem *pLast; /* Last field of the record */ - int nField; /* Number of fields in the record */ - char *zAffinity; /* The affinity string for the record */ - int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] */ - int len; /* Length of a field */ - } ao; - struct OP_Count_stack_vars { - i64 nEntry; - BtCursor *pCrsr; - } ap; - struct OP_Savepoint_stack_vars { - int p1; /* Value of P1 operand */ - char *zName; /* Name of savepoint */ - int nName; - Savepoint *pNew; - Savepoint *pSavepoint; - Savepoint *pTmp; - int iSavepoint; - int ii; - } aq; - struct OP_AutoCommit_stack_vars { - int desiredAutoCommit; - int iRollback; - int turnOnAC; - } ar; - struct OP_Transaction_stack_vars { - Btree *pBt; - } as; - struct OP_ReadCookie_stack_vars { - int iMeta; - int iDb; - int iCookie; - } at; - struct OP_SetCookie_stack_vars { - Db *pDb; - } au; - struct OP_VerifyCookie_stack_vars { - int iMeta; - Btree *pBt; - } av; - struct OP_OpenWrite_stack_vars { - int nField; - KeyInfo *pKeyInfo; - int p2; - int iDb; - int wrFlag; - Btree *pX; - VdbeCursor *pCur; - Db *pDb; - } aw; - struct OP_OpenEphemeral_stack_vars { - VdbeCursor *pCx; - } ax; - struct OP_OpenPseudo_stack_vars { - VdbeCursor *pCx; - } ay; - struct OP_SeekGt_stack_vars { - int res; - int oc; - VdbeCursor *pC; - UnpackedRecord r; - int nField; - i64 iKey; /* The rowid we are to seek to */ - } az; - struct OP_Seek_stack_vars { - VdbeCursor *pC; - } ba; - struct OP_Found_stack_vars { - int alreadyExists; - VdbeCursor *pC; - int res; - UnpackedRecord *pIdxKey; - UnpackedRecord r; - char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; - } bb; - struct OP_IsUnique_stack_vars { - u16 ii; - VdbeCursor *pCx; - BtCursor *pCrsr; - u16 nField; - Mem *aMx; - UnpackedRecord r; /* B-Tree index search key */ - i64 R; /* Rowid stored in register P3 */ - } bc; - struct OP_NotExists_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - u64 iKey; - } bd; - struct OP_NewRowid_stack_vars { - i64 v; /* The new rowid */ - VdbeCursor *pC; /* Cursor of table to get the new rowid */ - int res; /* Result of an sqlite3BtreeLast() */ - int cnt; /* Counter to limit the number of searches */ - Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ - VdbeFrame *pFrame; /* Root frame of VDBE */ - } be; - struct OP_InsertInt_stack_vars { - Mem *pData; /* MEM cell holding data for the record to be inserted */ - Mem *pKey; /* MEM cell holding key for the record */ - i64 iKey; /* The integer ROWID or key for the record to be inserted */ - VdbeCursor *pC; /* Cursor to table into which insert is written */ - int nZero; /* Number of zero-bytes to append */ - int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ - const char *zDb; /* database name - used by the update hook */ - const char *zTbl; /* Table name - used by the opdate hook */ - int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ - } bf; - struct OP_Delete_stack_vars { - i64 iKey; - VdbeCursor *pC; - } bg; - struct OP_RowData_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - u32 n; - i64 n64; - } bh; - struct OP_Rowid_stack_vars { - VdbeCursor *pC; - i64 v; - sqlite3_vtab *pVtab; - const sqlite3_module *pModule; - } bi; - struct OP_NullRow_stack_vars { - VdbeCursor *pC; - } bj; - struct OP_Last_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - } bk; - struct OP_Rewind_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - } bl; - struct OP_Next_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - } bm; - struct OP_IdxInsert_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - int nKey; - const char *zKey; - } bn; - struct OP_IdxDelete_stack_vars { - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - UnpackedRecord r; - } bo; - struct OP_IdxRowid_stack_vars { - BtCursor *pCrsr; - VdbeCursor *pC; - i64 rowid; - } bp; - struct OP_IdxGE_stack_vars { - VdbeCursor *pC; - int res; - UnpackedRecord r; - } bq; - struct OP_Destroy_stack_vars { - int iMoved; - int iCnt; - Vdbe *pVdbe; - int iDb; - } br; - struct OP_Clear_stack_vars { - int nChange; - } bs; - struct OP_CreateTable_stack_vars { - int pgno; - int flags; - Db *pDb; - } bt; - struct OP_ParseSchema_stack_vars { - int iDb; - const char *zMaster; - char *zSql; - InitData initData; - } bu; - struct OP_IntegrityCk_stack_vars { - int nRoot; /* Number of tables to check. (Number of root pages.) */ - int *aRoot; /* Array of rootpage numbers for tables to be checked */ - int j; /* Loop counter */ - int nErr; /* Number of errors reported */ - char *z; /* Text of the error report */ - Mem *pnErr; /* Register keeping track of errors remaining */ - } bv; - struct OP_RowSetRead_stack_vars { - i64 val; - } bw; - struct OP_RowSetTest_stack_vars { - int iSet; - int exists; - } bx; - struct OP_Program_stack_vars { - int nMem; /* Number of memory registers for sub-program */ - int nByte; /* Bytes of runtime space required for sub-program */ - Mem *pRt; /* Register to allocate runtime space */ - Mem *pMem; /* Used to iterate through memory cells */ - Mem *pEnd; /* Last memory cell in new array */ - VdbeFrame *pFrame; /* New vdbe frame to execute in */ - SubProgram *pProgram; /* Sub-program to execute */ - void *t; /* Token identifying trigger */ - } by; - struct OP_Param_stack_vars { - VdbeFrame *pFrame; - Mem *pIn; - } bz; - struct OP_MemMax_stack_vars { - Mem *pIn1; - VdbeFrame *pFrame; - } ca; - struct OP_AggStep_stack_vars { - int n; - int i; - Mem *pMem; - Mem *pRec; - sqlite3_context ctx; - sqlite3_value **apVal; - } cb; - struct OP_AggFinal_stack_vars { - Mem *pMem; - } cc; - struct OP_JournalMode_stack_vars { - Btree *pBt; /* Btree to change journal mode of */ - Pager *pPager; /* Pager associated with pBt */ - int eNew; /* New journal mode */ - int eOld; /* The old journal mode */ - const char *zFilename; /* Name of database file for pPager */ - } cd; - struct OP_IncrVacuum_stack_vars { - Btree *pBt; - } ce; - struct OP_VBegin_stack_vars { - VTable *pVTab; - } cf; - struct OP_VOpen_stack_vars { - VdbeCursor *pCur; - sqlite3_vtab_cursor *pVtabCursor; - sqlite3_vtab *pVtab; - sqlite3_module *pModule; - } cg; - struct OP_VFilter_stack_vars { - int nArg; - int iQuery; - const sqlite3_module *pModule; - Mem *pQuery; - Mem *pArgc; - sqlite3_vtab_cursor *pVtabCursor; - sqlite3_vtab *pVtab; - VdbeCursor *pCur; - int res; - int i; - Mem **apArg; - } ch; - struct OP_VColumn_stack_vars { - sqlite3_vtab *pVtab; - const sqlite3_module *pModule; - Mem *pDest; - sqlite3_context sContext; - } ci; - struct OP_VNext_stack_vars { - sqlite3_vtab *pVtab; - const sqlite3_module *pModule; - int res; - VdbeCursor *pCur; - } cj; - struct OP_VRename_stack_vars { - sqlite3_vtab *pVtab; - Mem *pName; - } ck; - struct OP_VUpdate_stack_vars { - sqlite3_vtab *pVtab; - sqlite3_module *pModule; - int nArg; - int i; - sqlite_int64 rowid; - Mem **apArg; - Mem *pX; - } cl; - struct OP_Trace_stack_vars { - char *zTrace; - } cm; - } u; - /* End automatically generated code - ********************************************************************/ - - assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ - sqlite3VdbeMutexArrayEnter(p); - if( p->rc==SQLITE_NOMEM ){ - /* This happens if a malloc() inside a call to sqlite3_column_text() or - ** sqlite3_column_text16() failed. */ - goto no_mem; +SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetInt64(&pCtx->s, (i64)iVal); +} +SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetInt64(&pCtx->s, iVal); +} +SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetNull(&pCtx->s); +} +SQLITE_API void sqlite3_result_text( + sqlite3_context *pCtx, + const char *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API void sqlite3_result_text16( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); +} +SQLITE_API void sqlite3_result_text16be( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); +} +SQLITE_API void sqlite3_result_text16le( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemCopy(&pCtx->s, pValue); +} +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetZeroBlob(&pCtx->s, n); +} +SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ + pCtx->isError = errCode; + if( pCtx->s.flags & MEM_Null ){ + sqlite3VdbeMemSetStr(&pCtx->s, sqlite3ErrStr(errCode), -1, + SQLITE_UTF8, SQLITE_STATIC); } - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); - p->rc = SQLITE_OK; - assert( p->explain==0 ); - p->pResultSet = 0; - db->busyHandler.nBusy = 0; - CHECK_FOR_INTERRUPT; - sqlite3VdbeIOTraceSql(p); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - checkProgress = db->xProgress!=0; -#endif -#ifdef SQLITE_DEBUG - sqlite3BeginBenignMalloc(); - if( p->pc==0 && (p->db->flags & SQLITE_VdbeListing)!=0 ){ - int i; - printf("VDBE Program Listing:\n"); - sqlite3VdbePrintSql(p); - for(i=0; inOp; i++){ - sqlite3VdbePrintOp(stdout, i, &aOp[i]); +} + +/* Force an SQLITE_TOOBIG error. */ +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pCtx->isError = SQLITE_TOOBIG; + sqlite3VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, + SQLITE_UTF8, SQLITE_STATIC); +} + +/* An SQLITE_NOMEM error. */ +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + sqlite3VdbeMemSetNull(&pCtx->s); + pCtx->isError = SQLITE_NOMEM; + pCtx->s.db->mallocFailed = 1; +} + +/* +** This function is called after a transaction has been committed. It +** invokes callbacks registered with sqlite3_wal_hook() as required. +*/ +static int doWalCallbacks(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_WAL + int i; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ + rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); + } } } - sqlite3EndBenignMalloc(); -#endif - for(pc=p->pc; rc==SQLITE_OK; pc++){ - assert( pc>=0 && pcnOp ); - if( db->mallocFailed ) goto no_mem; -#ifdef VDBE_PROFILE - origPc = pc; - start = sqlite3Hwtime(); #endif - pOp = &aOp[pc]; + return rc; +} - /* Only allow tracing if SQLITE_DEBUG is defined. +/* +** Execute the statement pStmt, either until a row of data is ready, the +** statement is completely executed or an error occurs. +** +** This routine implements the bulk of the logic behind the sqlite_step() +** API. The only thing omitted is the automatic recompile if a +** schema change has occurred. That detail is handled by the +** outer sqlite3_step() wrapper procedure. +*/ +static int sqlite3Step(Vdbe *p){ + sqlite3 *db; + int rc; + + assert(p); + if( p->magic!=VDBE_MAGIC_RUN ){ + /* We used to require that sqlite3_reset() be called before retrying + ** sqlite3_step() after any error or after SQLITE_DONE. But beginning + ** with version 3.7.0, we changed this so that sqlite3_reset() would + ** be called automatically instead of throwing the SQLITE_MISUSE error. + ** This "automatic-reset" change is not technically an incompatibility, + ** since any application that receives an SQLITE_MISUSE is broken by + ** definition. + ** + ** Nevertheless, some published applications that were originally written + ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE + ** returns, and the so were broken by the automatic-reset change. As a + ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the + ** legacy behavior of returning SQLITE_MISUSE for cases where the + ** previous sqlite3_step() returned something other than a SQLITE_LOCKED + ** or SQLITE_BUSY error. */ -#ifdef SQLITE_DEBUG - if( p->trace ){ - if( pc==0 ){ - printf("VDBE Execution Trace:\n"); - sqlite3VdbePrintSql(p); - } - sqlite3VdbePrintOp(p->trace, pc, pOp); +#ifdef SQLITE_OMIT_AUTORESET + if( p->rc==SQLITE_BUSY || p->rc==SQLITE_LOCKED ){ + sqlite3_reset((sqlite3_stmt*)p); + }else{ + return SQLITE_MISUSE_BKPT; } +#else + sqlite3_reset((sqlite3_stmt*)p); #endif - + } - /* Check to see if we need to simulate an interrupt. This only happens - ** if we have a special test build. + /* Check that malloc() has not failed. If it has, return early. */ + db = p->db; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + + if( p->pc<=0 && p->expired ){ + p->rc = SQLITE_SCHEMA; + rc = SQLITE_ERROR; + goto end_of_step; + } + if( p->pc<0 ){ + /* If there are no other statements currently running, then + ** reset the interrupt flag. This prevents a call to sqlite3_interrupt + ** from interrupting a statement that has not yet started. */ -#ifdef SQLITE_TEST - if( sqlite3_interrupt_count>0 ){ - sqlite3_interrupt_count--; - if( sqlite3_interrupt_count==0 ){ - sqlite3_interrupt(db); - } + if( db->activeVdbeCnt==0 ){ + db->u1.isInterrupted = 0; } -#endif -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Call the progress callback if it is configured and the required number - ** of VDBE ops have been executed (either since this invocation of - ** sqlite3VdbeExec() or since last time the progress callback was called). - ** If the progress callback returns non-zero, exit the virtual machine with - ** a return code SQLITE_ABORT. - */ - if( checkProgress ){ - if( db->nProgressOps==nProgressOps ){ - int prc; - prc = db->xProgress(db->pProgressArg); - if( prc!=0 ){ - rc = SQLITE_INTERRUPT; - goto vdbe_error_halt; - } - nProgressOps = 0; - } - nProgressOps++; + assert( db->writeVdbeCnt>0 || db->autoCommit==0 || db->nDeferredCons==0 ); + +#ifndef SQLITE_OMIT_TRACE + if( db->xProfile && !db->init.busy ){ + sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); } #endif - /* On any opcode with the "out2-prerelase" tag, free any - ** external allocations out of mem[p2] and set mem[p2] to be - ** an undefined integer. Opcodes will either fill in the integer - ** value or convert mem[p2] to a different type. - */ - assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); - if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); - sqlite3VdbeMemReleaseExternal(pOut); - pOut->flags = MEM_Int; - } + db->activeVdbeCnt++; + if( p->readOnly==0 ) db->writeVdbeCnt++; + p->pc = 0; + } +#ifndef SQLITE_OMIT_EXPLAIN + if( p->explain ){ + rc = sqlite3VdbeList(p); + }else +#endif /* SQLITE_OMIT_EXPLAIN */ + { + db->vdbeExecCnt++; + rc = sqlite3VdbeExec(p); + db->vdbeExecCnt--; + } - /* Sanity checking on other operands */ -#ifdef SQLITE_DEBUG - if( (pOp->opflags & OPFLG_IN1)!=0 ){ - assert( pOp->p1>0 ); - assert( pOp->p1<=p->nMem ); - assert( memIsValid(&aMem[pOp->p1]) ); - REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); - } - if( (pOp->opflags & OPFLG_IN2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - assert( memIsValid(&aMem[pOp->p2]) ); - REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); - } - if( (pOp->opflags & OPFLG_IN3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); - assert( memIsValid(&aMem[pOp->p3]) ); - REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); - } - if( (pOp->opflags & OPFLG_OUT2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - memAboutToChange(p, &aMem[pOp->p2]); - } - if( (pOp->opflags & OPFLG_OUT3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); - memAboutToChange(p, &aMem[pOp->p3]); +#ifndef SQLITE_OMIT_TRACE + /* Invoke the profile callback if there is one + */ + if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ + sqlite3_int64 iNow; + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); + } +#endif + + if( rc==SQLITE_DONE ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; } + } + + db->errCode = rc; + if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ + p->rc = SQLITE_NOMEM; + } +end_of_step: + /* At this point local variable rc holds the value that should be + ** returned if this statement was compiled using the legacy + ** sqlite3_prepare() interface. According to the docs, this can only + ** be one of the values in the first assert() below. Variable p->rc + ** contains the value that would be returned if sqlite3_finalize() + ** were called on statement p. + */ + assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR + || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + ); + assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); + if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ + /* If this statement was prepared using sqlite3_prepare_v2(), and an + ** error has occured, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same value. + */ + rc = db->errCode = p->rc; + } + return (rc&db->errMask); +} + +/* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 5 #endif - - switch( pOp->opcode ){ -/***************************************************************************** -** What follows is a massive switch statement where each case implements a -** separate instruction in the virtual machine. If we follow the usual -** indentation conventions, each case should be indented by 6 spaces. But -** that is a lot of wasted space on the left margin. So the code within -** the switch statement will break with convention and be flush-left. Another -** big comment (similar to this one) will mark the point in the code where -** we transition back to normal indentation. -** -** The formatting of each case is important. The makefile for SQLite -** generates two C files "opcodes.h" and "opcodes.c" by scanning this -** file looking for lines that begin with "case OP_". The opcodes.h files -** will be filled with #defines that give unique integer values to each -** opcode and the opcodes.c file is filled with an array of strings where -** each string is the symbolic name for the corresponding opcode. If the -** case statement is followed by a comment of the form "/# same as ... #/" -** that comment is used to determine the particular value of the opcode. -** -** Other keywords in the comment that follows each case are used to -** construct the OPFLG_INITIALIZER value that initializes opcodeProperty[]. -** Keywords include: in1, in2, in3, out2_prerelease, out2, out3. See -** the mkopcodeh.awk script for additional information. -** -** Documentation about VDBE opcodes is generated by scanning this file -** for lines of that contain "Opcode:". That line and all subsequent -** comment lines are used in the generation of the opcode.html documentation -** file. -** -** SUMMARY: -** -** Formatting is important to scripts that scan this file. -** Do not deviate from the formatting style currently in use. -** -*****************************************************************************/ +/* +** This is the top-level implementation of sqlite3_step(). Call +** sqlite3Step() to do most of the work. If a schema error occurs, +** call sqlite3Reprepare() and try again. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ + int rc = SQLITE_OK; /* Result from sqlite3Step() */ + int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ + Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ + int cnt = 0; /* Counter to prevent infinite loop of reprepares */ + sqlite3 *db; /* The database connection */ -/* Opcode: Goto * P2 * * * -** -** An unconditional jump to address P2. -** The next instruction executed will be -** the one at index P2 from the beginning of -** the program. + if( vdbeSafetyNotNull(v) ){ + return SQLITE_MISUSE_BKPT; + } + db = v->db; + sqlite3_mutex_enter(db->mutex); + while( (rc = sqlite3Step(v))==SQLITE_SCHEMA + && cnt++ < SQLITE_MAX_SCHEMA_RETRY + && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ + sqlite3_reset(pStmt); + v->expired = 0; + } + if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ + /* This case occurs after failing to recompile an sql statement. + ** The error message from the SQL compiler has already been loaded + ** into the database handle. This block copies the error message + ** from the database handle into the statement and sets the statement + ** program counter to 0 to ensure that when the statement is + ** finalized or reset the parser error message is available via + ** sqlite3_errmsg() and sqlite3_errcode(). + */ + const char *zErr = (const char *)sqlite3_value_text(db->pErr); + sqlite3DbFree(db, v->zErrMsg); + if( !db->mallocFailed ){ + v->zErrMsg = sqlite3DbStrDup(db, zErr); + v->rc = rc2; + } else { + v->zErrMsg = 0; + v->rc = rc = SQLITE_NOMEM; + } + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Extract the user data from a sqlite3_context structure and return a +** pointer to it. */ -case OP_Goto: { /* jump */ - CHECK_FOR_INTERRUPT; - pc = pOp->p2 - 1; - break; +SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ + assert( p && p->pFunc ); + return p->pFunc->pUserData; } -/* Opcode: Gosub P1 P2 * * * +/* +** Extract the user data from a sqlite3_context structure and return a +** pointer to it. ** -** Write the current address onto register P1 -** and then jump to address P2. +** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface +** returns a copy of the pointer to the database connection (the 1st +** parameter) of the sqlite3_create_function() and +** sqlite3_create_function16() routines that originally registered the +** application defined function. */ -case OP_Gosub: { /* jump, in1 */ - pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Dyn)==0 ); - memAboutToChange(p, pIn1); - pIn1->flags = MEM_Int; - pIn1->u.i = pc; - REGISTER_TRACE(pOp->p1, pIn1); - pc = pOp->p2 - 1; - break; +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ + assert( p && p->pFunc ); + return p->s.db; } -/* Opcode: Return P1 * * * * -** -** Jump to the next instruction after the address in register P1. +/* +** The following is the implementation of an SQL function that always +** fails with an error message stating that the function is used in the +** wrong context. The sqlite3_overload_function() API might construct +** SQL function that use this routine so that the functions will exist +** for name resolution but are actually overloaded by the xFindFunction +** method of virtual tables. */ -case OP_Return: { /* in1 */ - pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags & MEM_Int ); - pc = (int)pIn1->u.i; - break; +SQLITE_PRIVATE void sqlite3InvalidFunction( + sqlite3_context *context, /* The function calling context */ + int NotUsed, /* Number of arguments to the function */ + sqlite3_value **NotUsed2 /* Value of each argument */ +){ + const char *zName = context->pFunc->zName; + char *zErr; + UNUSED_PARAMETER2(NotUsed, NotUsed2); + zErr = sqlite3_mprintf( + "unable to use function %s in the requested context", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); } -/* Opcode: Yield P1 * * * * -** -** Swap the program counter with the value in register P1. +/* +** Allocate or return the aggregate context for a user function. A new +** context is allocated on the first call. Subsequent calls return the +** same context that was returned on prior calls. */ -case OP_Yield: { /* in1 */ -#if 0 /* local variables moved into u.aa */ - int pcDest; -#endif /* local variables moved into u.aa */ - pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Dyn)==0 ); - pIn1->flags = MEM_Int; - u.aa.pcDest = (int)pIn1->u.i; - pIn1->u.i = pc; - REGISTER_TRACE(pOp->p1, pIn1); - pc = u.aa.pcDest; - break; +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ + Mem *pMem; + assert( p && p->pFunc && p->pFunc->xStep ); + assert( sqlite3_mutex_held(p->s.db->mutex) ); + pMem = p->pMem; + testcase( nByte<0 ); + if( (pMem->flags & MEM_Agg)==0 ){ + if( nByte<=0 ){ + sqlite3VdbeMemReleaseExternal(pMem); + pMem->flags = MEM_Null; + pMem->z = 0; + }else{ + sqlite3VdbeMemGrow(pMem, nByte, 0); + pMem->flags = MEM_Agg; + pMem->u.pDef = p->pFunc; + if( pMem->z ){ + memset(pMem->z, 0, nByte); + } + } + } + return (void*)pMem->z; } -/* Opcode: HaltIfNull P1 P2 P3 P4 * -** -** Check the value in register P3. If is is NULL then Halt using -** parameter P1, P2, and P4 as if this were a Halt instruction. If the -** value in register P3 is not NULL, then this routine is a no-op. +/* +** Return the auxilary data pointer, if any, for the iArg'th argument to +** the user-function defined by pCtx. */ -case OP_HaltIfNull: { /* in3 */ - pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & MEM_Null)==0 ) break; - /* Fall through into OP_Halt */ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ + VdbeFunc *pVdbeFunc; + + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pVdbeFunc = pCtx->pVdbeFunc; + if( !pVdbeFunc || iArg>=pVdbeFunc->nAux || iArg<0 ){ + return 0; + } + return pVdbeFunc->apAux[iArg].pAux; } -/* Opcode: Halt P1 P2 * P4 * -** -** Exit immediately. All open cursors, etc are closed -** automatically. -** -** P1 is the result code returned by sqlite3_exec(), sqlite3_reset(), -** or sqlite3_finalize(). For a normal halt, this should be SQLITE_OK (0). -** For errors, it can be some other value. If P1!=0 then P2 will determine -** whether or not to rollback the current transaction. Do not rollback -** if P2==OE_Fail. Do the rollback if P2==OE_Rollback. If P2==OE_Abort, -** then back out all changes that have occurred during this execution of the -** VDBE, but do not rollback the transaction. -** -** If P4 is not null then it is an error message string. -** -** There is an implied "Halt 0 0 0" instruction inserted at the very end of -** every program. So a jump past the last instruction of the program -** is the same as executing Halt. +/* +** Set the auxilary data pointer and delete function, for the iArg'th +** argument to the user-function defined by pCtx. Any previous value is +** deleted by calling the delete function specified when it was set. */ -case OP_Halt: { - if( pOp->p1==SQLITE_OK && p->pFrame ){ - /* Halt the sub-program. Return control to the parent frame. */ - VdbeFrame *pFrame = p->pFrame; - p->pFrame = pFrame->pParent; - p->nFrame--; - sqlite3VdbeSetChanges(db, p->nChange); - pc = sqlite3VdbeFrameRestore(pFrame); - if( pOp->p2==OE_Ignore ){ - /* Instruction pc is the OP_Program that invoked the sub-program - ** currently being halted. If the p2 instruction of this OP_Halt - ** instruction is set to OE_Ignore, then the sub-program is throwing - ** an IGNORE exception. In this case jump to the address specified - ** as the p2 of the calling OP_Program. */ - pc = p->aOp[pc].p2-1; +SQLITE_API void sqlite3_set_auxdata( + sqlite3_context *pCtx, + int iArg, + void *pAux, + void (*xDelete)(void*) +){ + struct AuxData *pAuxData; + VdbeFunc *pVdbeFunc; + if( iArg<0 ) goto failed; + + assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + pVdbeFunc = pCtx->pVdbeFunc; + if( !pVdbeFunc || pVdbeFunc->nAux<=iArg ){ + int nAux = (pVdbeFunc ? pVdbeFunc->nAux : 0); + int nMalloc = sizeof(VdbeFunc) + sizeof(struct AuxData)*iArg; + pVdbeFunc = sqlite3DbRealloc(pCtx->s.db, pVdbeFunc, nMalloc); + if( !pVdbeFunc ){ + goto failed; } - aOp = p->aOp; - aMem = p->aMem; - break; + pCtx->pVdbeFunc = pVdbeFunc; + memset(&pVdbeFunc->apAux[nAux], 0, sizeof(struct AuxData)*(iArg+1-nAux)); + pVdbeFunc->nAux = iArg+1; + pVdbeFunc->pFunc = pCtx->pFunc; } - p->rc = pOp->p1; - p->errorAction = (u8)pOp->p2; - p->pc = pc; - if( pOp->p4.z ){ - assert( p->rc!=SQLITE_OK ); - sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pc, p->zSql, pOp->p4.z); - }else if( p->rc ){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(pOp->p1, "constraint failed at %d in [%s]", pc, p->zSql); + pAuxData = &pVdbeFunc->apAux[iArg]; + if( pAuxData->pAux && pAuxData->xDelete ){ + pAuxData->xDelete(pAuxData->pAux); } - rc = sqlite3VdbeHalt(p); - assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); - if( rc==SQLITE_BUSY ){ - p->rc = rc = SQLITE_BUSY; - }else{ - assert( rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ); - assert( rc==SQLITE_OK || db->nDeferredCons>0 ); - rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; + pAuxData->pAux = pAux; + pAuxData->xDelete = xDelete; + return; + +failed: + if( xDelete ){ + xDelete(pAux); } - goto vdbe_return; } -/* Opcode: Integer P1 P2 * * * +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Return the number of times the Step function of a aggregate has been +** called. ** -** The 32-bit integer value P1 is written into register P2. +** This function is deprecated. Do not use it for new code. It is +** provide only to avoid breaking legacy code. New aggregate function +** implementations should keep their own counts within their aggregate +** context. */ -case OP_Integer: { /* out2-prerelease */ - pOut->u.i = pOp->p1; - break; +SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ + assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); + return p->pMem->n; } +#endif -/* Opcode: Int64 * P2 * P4 * -** -** P4 is a pointer to a 64-bit integer value. -** Write that value into register P2. +/* +** Return the number of columns in the result set for the statement pStmt. */ -case OP_Int64: { /* out2-prerelease */ - assert( pOp->p4.pI64!=0 ); - pOut->u.i = *pOp->p4.pI64; - break; +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ + Vdbe *pVm = (Vdbe *)pStmt; + return pVm ? pVm->nResColumn : 0; } -#ifndef SQLITE_OMIT_FLOATING_POINT -/* Opcode: Real * P2 * P4 * -** -** P4 is a pointer to a 64-bit floating point value. -** Write that value into register P2. +/* +** Return the number of values available from the current row of the +** currently executing statement pStmt. */ -case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ - pOut->flags = MEM_Real; - assert( !sqlite3IsNaN(*pOp->p4.pReal) ); - pOut->r = *pOp->p4.pReal; - break; +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ + Vdbe *pVm = (Vdbe *)pStmt; + if( pVm==0 || pVm->pResultSet==0 ) return 0; + return pVm->nResColumn; } -#endif -/* Opcode: String8 * P2 * P4 * -** -** P4 points to a nul terminated UTF-8 string. This opcode is transformed -** into an OP_String before it is executed for the first time. -*/ -case OP_String8: { /* same as TK_STRING, out2-prerelease */ - assert( pOp->p4.z!=0 ); - pOp->opcode = OP_String; - pOp->p1 = sqlite3Strlen30(pOp->p4.z); -#ifndef SQLITE_OMIT_UTF16 - if( encoding!=SQLITE_UTF8 ){ - rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); - if( rc==SQLITE_TOOBIG ) goto too_big; - if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; - assert( pOut->zMalloc==pOut->z ); - assert( pOut->flags & MEM_Dyn ); - pOut->zMalloc = 0; - pOut->flags |= MEM_Static; - pOut->flags &= ~MEM_Dyn; - if( pOp->p4type==P4_DYNAMIC ){ - sqlite3DbFree(db, pOp->p4.z); - } - pOp->p4type = P4_DYNAMIC; - pOp->p4.z = pOut->z; - pOp->p1 = pOut->n; - } +/* +** Check to see if column iCol of the given statement is valid. If +** it is, return a pointer to the Mem for the value of that column. +** If iCol is not valid, return a pointer to a Mem which has a value +** of NULL. +*/ +static Mem *columnMem(sqlite3_stmt *pStmt, int i){ + Vdbe *pVm; + Mem *pOut; + + pVm = (Vdbe *)pStmt; + if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){ + sqlite3_mutex_enter(pVm->db->mutex); + pOut = &pVm->pResultSet[i]; + }else{ + /* If the value passed as the second argument is out of range, return + ** a pointer to the following static Mem object which contains the + ** value SQL NULL. Even though the Mem structure contains an element + ** of type i64, on certain architecture (x86) with certain compiler + ** switches (-Os), gcc may align this Mem object on a 4-byte boundary + ** instead of an 8-byte one. This all works fine, except that when + ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s + ** that a Mem structure is located on an 8-byte boundary. To prevent + ** this assert() from failing, when building with SQLITE_DEBUG defined + ** using gcc, force nullMem to be 8-byte aligned using the magical + ** __attribute__((aligned(8))) macro. */ + static const Mem nullMem +#if defined(SQLITE_DEBUG) && defined(__GNUC__) + __attribute__((aligned(8))) #endif - if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; + = {0, "", (double)0, {0}, 0, MEM_Null, SQLITE_NULL, 0, +#ifdef SQLITE_DEBUG + 0, 0, /* pScopyFrom, pFiller */ +#endif + 0, 0 }; + + if( pVm && ALWAYS(pVm->db) ){ + sqlite3_mutex_enter(pVm->db->mutex); + sqlite3Error(pVm->db, SQLITE_RANGE, 0); + } + pOut = (Mem*)&nullMem; } - /* Fall through to the next case, OP_String */ + return pOut; } - -/* Opcode: String P1 P2 * P4 * + +/* +** This function is called after invoking an sqlite3_value_XXX function on a +** column value (i.e. a value returned by evaluating an SQL expression in the +** select list of a SELECT statement) that may cause a malloc() failure. If +** malloc() has failed, the threads mallocFailed flag is cleared and the result +** code of statement pStmt set to SQLITE_NOMEM. ** -** The string value P4 of length P1 (bytes) is stored in register P2. +** Specifically, this is called from within: +** +** sqlite3_column_int() +** sqlite3_column_int64() +** sqlite3_column_text() +** sqlite3_column_text16() +** sqlite3_column_real() +** sqlite3_column_bytes() +** sqlite3_column_bytes16() +** sqiite3_column_blob() */ -case OP_String: { /* out2-prerelease */ - assert( pOp->p4.z!=0 ); - pOut->flags = MEM_Str|MEM_Static|MEM_Term; - pOut->z = pOp->p4.z; - pOut->n = pOp->p1; - pOut->enc = encoding; - UPDATE_MAX_BLOBSIZE(pOut); - break; +static void columnMallocFailure(sqlite3_stmt *pStmt) +{ + /* If malloc() failed during an encoding conversion within an + ** sqlite3_column_XXX API, then set the return code of the statement to + ** SQLITE_NOMEM. The next call to _step() (if any) will return SQLITE_ERROR + ** and _finalize() will return NOMEM. + */ + Vdbe *p = (Vdbe *)pStmt; + if( p ){ + p->rc = sqlite3ApiExit(p->db, p->rc); + sqlite3_mutex_leave(p->db->mutex); + } } -/* Opcode: Null * P2 * * * -** -** Write a NULL into register P2. +/**************************** sqlite3_column_ ******************************* +** The following routines are used to access elements of the current row +** in the result set. */ -case OP_Null: { /* out2-prerelease */ - pOut->flags = MEM_Null; - break; +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ + const void *val; + val = sqlite3_value_blob( columnMem(pStmt,i) ); + /* Even though there is no encoding conversion, value_blob() might + ** need to call malloc() to expand the result of a zeroblob() + ** expression. + */ + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_bytes( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ + double val = sqlite3_value_double( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_int( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ + sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ + const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ + Mem *pOut = columnMem(pStmt, i); + if( pOut->flags&MEM_Static ){ + pOut->flags &= ~MEM_Static; + pOut->flags |= MEM_Ephem; + } + columnMallocFailure(pStmt); + return (sqlite3_value *)pOut; +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ + const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ + int iType = sqlite3_value_type( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return iType; } - -/* Opcode: Blob P1 P2 * P4 -** -** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. +/* The following function is experimental and subject to change or +** removal */ +/*int sqlite3_column_numeric_type(sqlite3_stmt *pStmt, int i){ +** return sqlite3_value_numeric_type( columnMem(pStmt,i) ); +**} */ -case OP_Blob: { /* out2-prerelease */ - assert( pOp->p1 <= SQLITE_MAX_LENGTH ); - sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); - pOut->enc = encoding; - UPDATE_MAX_BLOBSIZE(pOut); - break; -} -/* Opcode: Variable P1 P2 * P4 * +/* +** Convert the N-th element of pStmt->pColName[] into a string using +** xFunc() then return that string. If N is out of range, return 0. ** -** Transfer the values of bound parameter P1 into register P2 +** There are up to 5 names for each column. useType determines which +** name is returned. Here are the names: ** -** If the parameter is named, then its name appears in P4 and P3==1. -** The P4 value is used by sqlite3_bind_parameter_name(). +** 0 The column name as it should be displayed for output +** 1 The datatype name for the column +** 2 The name of the database that the column derives from +** 3 The name of the table that the column derives from +** 4 The name of the table column that the result column derives from +** +** If the result is not a simple column reference (if it is an expression +** or a constant) then useTypes 2, 3, and 4 return NULL. */ -case OP_Variable: { /* out2-prerelease */ -#if 0 /* local variables moved into u.ab */ - Mem *pVar; /* Value being transferred */ -#endif /* local variables moved into u.ab */ - - assert( pOp->p1>0 && pOp->p1<=p->nVar ); - u.ab.pVar = &p->aVar[pOp->p1 - 1]; - if( sqlite3VdbeMemTooBig(u.ab.pVar) ){ - goto too_big; +static const void *columnName( + sqlite3_stmt *pStmt, + int N, + const void *(*xFunc)(Mem*), + int useType +){ + const void *ret = 0; + Vdbe *p = (Vdbe *)pStmt; + int n; + sqlite3 *db = p->db; + + assert( db!=0 ); + n = sqlite3_column_count(pStmt); + if( N=0 ){ + N += useType*n; + sqlite3_mutex_enter(db->mutex); + assert( db->mallocFailed==0 ); + ret = xFunc(&p->aColName[N]); + /* A malloc may have failed inside of the xFunc() call. If this + ** is the case, clear the mallocFailed flag and return NULL. + */ + if( db->mallocFailed ){ + db->mallocFailed = 0; + ret = 0; + } + sqlite3_mutex_leave(db->mutex); } - sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static); - UPDATE_MAX_BLOBSIZE(pOut); - break; + return ret; } -/* Opcode: Move P1 P2 P3 * * -** -** Move the values in register P1..P1+P3-1 over into -** registers P2..P2+P3-1. Registers P1..P1+P1-1 are -** left holding a NULL. It is an error for register ranges -** P1..P1+P3-1 and P2..P2+P3-1 to overlap. +/* +** Return the name of the Nth column of the result set returned by SQL +** statement pStmt. */ -case OP_Move: { -#if 0 /* local variables moved into u.ac */ - char *zMalloc; /* Holding variable for allocated memory */ - int n; /* Number of registers left to copy */ - int p1; /* Register to copy from */ - int p2; /* Register to copy to */ -#endif /* local variables moved into u.ac */ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); +} +#endif - u.ac.n = pOp->p3; - u.ac.p1 = pOp->p1; - u.ac.p2 = pOp->p2; - assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 ); - assert( u.ac.p1+u.ac.n<=u.ac.p2 || u.ac.p2+u.ac.n<=u.ac.p1 ); +/* +** Constraint: If you have ENABLE_COLUMN_METADATA then you must +** not define OMIT_DECLTYPE. +*/ +#if defined(SQLITE_OMIT_DECLTYPE) && defined(SQLITE_ENABLE_COLUMN_METADATA) +# error "Must not define both SQLITE_OMIT_DECLTYPE \ + and SQLITE_ENABLE_COLUMN_METADATA" +#endif - pIn1 = &aMem[u.ac.p1]; - pOut = &aMem[u.ac.p2]; - while( u.ac.n-- ){ - assert( pOut<=&aMem[p->nMem] ); - assert( pIn1<=&aMem[p->nMem] ); - assert( memIsValid(pIn1) ); - memAboutToChange(p, pOut); - u.ac.zMalloc = pOut->zMalloc; - pOut->zMalloc = 0; - sqlite3VdbeMemMove(pOut, pIn1); - pIn1->zMalloc = u.ac.zMalloc; - REGISTER_TRACE(u.ac.p2++, pOut); - pIn1++; - pOut++; - } - break; +#ifndef SQLITE_OMIT_DECLTYPE +/* +** Return the column declaration type (if applicable) of the 'i'th column +** of the result set of SQL statement pStmt. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); } +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_DECLTYPE */ -/* Opcode: Copy P1 P2 * * * -** -** Make a copy of register P1 into register P2. -** -** This instruction makes a deep copy of the value. A duplicate -** is made of any string or blob constant. See also OP_SCopy. +#ifdef SQLITE_ENABLE_COLUMN_METADATA +/* +** Return the name of the database from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unabiguous reference to a database column. */ -case OP_Copy: { /* in1, out2 */ - pIn1 = &aMem[pOp->p1]; - pOut = &aMem[pOp->p2]; - assert( pOut!=pIn1 ); - sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); - Deephemeralize(pOut); - REGISTER_TRACE(pOp->p2, pOut); - break; +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); } +#endif /* SQLITE_OMIT_UTF16 */ -/* Opcode: SCopy P1 P2 * * * -** -** Make a shallow copy of register P1 into register P2. -** -** This instruction makes a shallow copy of the value. If the value -** is a string or blob, then the copy is only a pointer to the -** original and hence if the original changes so will the copy. -** Worse, if the original is deallocated, the copy becomes invalid. -** Thus the program must guarantee that the original will not change -** during the lifetime of the copy. Use OP_Copy to make a complete -** copy. +/* +** Return the name of the table from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unabiguous reference to a database column. */ -case OP_SCopy: { /* in1, out2 */ - pIn1 = &aMem[pOp->p1]; - pOut = &aMem[pOp->p2]; - assert( pOut!=pIn1 ); - sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); -#ifdef SQLITE_DEBUG - if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; -#endif - REGISTER_TRACE(pOp->p2, pOut); - break; +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); } +#endif /* SQLITE_OMIT_UTF16 */ -/* Opcode: ResultRow P1 P2 * * * -** -** The registers P1 through P1+P2-1 contain a single row of -** results. This opcode causes the sqlite3_step() call to terminate -** with an SQLITE_ROW return code and it sets up the sqlite3_stmt -** structure to provide access to the top P1 values as the result -** row. +/* +** Return the name of the table column from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unabiguous reference to a database column. */ -case OP_ResultRow: { -#if 0 /* local variables moved into u.ad */ - Mem *pMem; - int i; -#endif /* local variables moved into u.ad */ - assert( p->nResColumn==pOp->p2 ); - assert( pOp->p1>0 ); - assert( pOp->p1+pOp->p2<=p->nMem+1 ); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ + return columnName( + pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_ENABLE_COLUMN_METADATA */ - /* If this statement has violated immediate foreign key constraints, do - ** not return the number of rows modified. And do not RELEASE the statement - ** transaction. It needs to be rolled back. */ - if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ - assert( db->flags&SQLITE_CountRows ); - assert( p->usesStmtJournal ); - break; - } - /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then - ** DML statements invoke this opcode to return the number of rows - ** modified to the user. This is the only way that a VM that - ** opens a statement transaction may invoke this opcode. - ** - ** In case this is such a statement, close any statement transaction - ** opened by this VM before returning control to the user. This is to - ** ensure that statement-transactions are always nested, not overlapping. - ** If the open statement-transaction is not closed here, then the user - ** may step another VM that opens its own statement transaction. This - ** may lead to overlapping statement transactions. - ** - ** The statement transaction is never a top-level transaction. Hence - ** the RELEASE call below can never fail. - */ - assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); - rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); - if( NEVER(rc!=SQLITE_OK) ){ - break; +/******************************* sqlite3_bind_ *************************** +** +** Routines used to attach values to wildcards in a compiled SQL statement. +*/ +/* +** Unbind the value bound to variable i in virtual machine p. This is the +** the same as binding a NULL value to the column. If the "i" parameter is +** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. +** +** A successful evaluation of this routine acquires the mutex on p. +** the mutex is released if any kind of error occurs. +** +** The error code stored in database p->db is overwritten with the return +** value in any case. +*/ +static int vdbeUnbind(Vdbe *p, int i){ + Mem *pVar; + if( vdbeSafetyNotNull(p) ){ + return SQLITE_MISUSE_BKPT; } - - /* Invalidate all ephemeral cursor row caches */ - p->cacheCtr = (p->cacheCtr + 2)|1; - - /* Make sure the results of the current row are \000 terminated - ** and have an assigned type. The results are de-ephemeralized as - ** as side effect. - */ - u.ad.pMem = p->pResultSet = &aMem[pOp->p1]; - for(u.ad.i=0; u.ad.ip2; u.ad.i++){ - assert( memIsValid(&u.ad.pMem[u.ad.i]) ); - Deephemeralize(&u.ad.pMem[u.ad.i]); - assert( (u.ad.pMem[u.ad.i].flags & MEM_Ephem)==0 - || (u.ad.pMem[u.ad.i].flags & (MEM_Str|MEM_Blob))==0 ); - sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]); - sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]); - REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]); + sqlite3_mutex_enter(p->db->mutex); + if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ + sqlite3Error(p->db, SQLITE_MISUSE, 0); + sqlite3_mutex_leave(p->db->mutex); + sqlite3_log(SQLITE_MISUSE, + "bind on a busy prepared statement: [%s]", p->zSql); + return SQLITE_MISUSE_BKPT; } - if( db->mallocFailed ) goto no_mem; + if( i<1 || i>p->nVar ){ + sqlite3Error(p->db, SQLITE_RANGE, 0); + sqlite3_mutex_leave(p->db->mutex); + return SQLITE_RANGE; + } + i--; + pVar = &p->aVar[i]; + sqlite3VdbeMemRelease(pVar); + pVar->flags = MEM_Null; + sqlite3Error(p->db, SQLITE_OK, 0); - /* Return SQLITE_ROW + /* If the bit corresponding to this variable in Vdbe.expmask is set, then + ** binding a new value to this variable invalidates the current query plan. + ** + ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call + ** following any change to the bindings of that parameter. */ - p->pc = pc + 1; - rc = SQLITE_ROW; - goto vdbe_return; + if( p->isPrepareV2 && + ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) + ){ + p->expired = 1; + } + return SQLITE_OK; } -/* Opcode: Concat P1 P2 P3 * * -** -** Add the text in register P1 onto the end of the text in -** register P2 and store the result in register P3. -** If either the P1 or P2 text are NULL then store NULL in P3. -** -** P3 = P2 || P1 -** -** It is illegal for P1 and P3 to be the same register. Sometimes, -** if P3 is the same register as P2, the implementation is able -** to avoid a memcpy(). +/* +** Bind a text or BLOB value. */ -case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ -#if 0 /* local variables moved into u.ae */ - i64 nByte; -#endif /* local variables moved into u.ae */ +static int bindText( + sqlite3_stmt *pStmt, /* The statement to bind against */ + int i, /* Index of the parameter to bind */ + const void *zData, /* Pointer to the data to be bound */ + int nData, /* Number of bytes of data to be bound */ + void (*xDel)(void*), /* Destructor for the data */ + u8 encoding /* Encoding for the data */ +){ + Vdbe *p = (Vdbe *)pStmt; + Mem *pVar; + int rc; - pIn1 = &aMem[pOp->p1]; - pIn2 = &aMem[pOp->p2]; - pOut = &aMem[pOp->p3]; - assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - break; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + if( zData!=0 ){ + pVar = &p->aVar[i-1]; + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( rc==SQLITE_OK && encoding!=0 ){ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + } + sqlite3Error(p->db, rc, 0); + rc = sqlite3ApiExit(p->db, rc); + } + sqlite3_mutex_leave(p->db->mutex); + }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ + xDel((void*)zData); } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); - u.ae.nByte = pIn1->n + pIn2->n; - if( u.ae.nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; + return rc; +} + + +/* +** Bind a blob value to an SQL statement variable. +*/ +SQLITE_API int sqlite3_bind_blob( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, 0); +} +SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); + sqlite3_mutex_leave(p->db->mutex); } - MemSetTypeFlag(pOut, MEM_Str); - if( sqlite3VdbeMemGrow(pOut, (int)u.ae.nByte+2, pOut==pIn2) ){ - goto no_mem; + return rc; +} +SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ + return sqlite3_bind_int64(p, i, (i64)iValue); +} +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); + sqlite3_mutex_leave(p->db->mutex); } - if( pOut!=pIn2 ){ - memcpy(pOut->z, pIn2->z, pIn2->n); + return rc; +} +SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ + int rc; + Vdbe *p = (Vdbe*)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3_mutex_leave(p->db->mutex); } - memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); - pOut->z[u.ae.nByte] = 0; - pOut->z[u.ae.nByte+1] = 0; - pOut->flags |= MEM_Term; - pOut->n = (int)u.ae.nByte; - pOut->enc = encoding; - UPDATE_MAX_BLOBSIZE(pOut); - break; + return rc; } - -/* Opcode: Add P1 P2 P3 * * -** -** Add the value in register P1 to the value in register P2 -** and store the result in register P3. -** If either input is NULL, the result is NULL. -*/ -/* Opcode: Multiply P1 P2 P3 * * -** -** -** Multiply the value in register P1 by the value in register P2 -** and store the result in register P3. -** If either input is NULL, the result is NULL. -*/ -/* Opcode: Subtract P1 P2 P3 * * -** -** Subtract the value in register P1 from the value in register P2 -** and store the result in register P3. -** If either input is NULL, the result is NULL. -*/ -/* Opcode: Divide P1 P2 P3 * * -** -** Divide the value in register P1 by the value in register P2 -** and store the result in register P3 (P3=P2/P1). If the value in -** register P1 is zero, then the result is NULL. If either input is -** NULL, the result is NULL. -*/ -/* Opcode: Remainder P1 P2 P3 * * -** -** Compute the remainder after integer division of the value in -** register P1 by the value in register P2 and store the result in P3. -** If the value in register P2 is zero the result is NULL. -** If either operand is NULL, the result is NULL. -*/ -case OP_Add: /* same as TK_PLUS, in1, in2, out3 */ -case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ -case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ -case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ -case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ -#if 0 /* local variables moved into u.af */ - int flags; /* Combined MEM_* flags from both inputs */ - i64 iA; /* Integer value of left operand */ - i64 iB; /* Integer value of right operand */ - double rA; /* Real value of left operand */ - double rB; /* Real value of right operand */ -#endif /* local variables moved into u.af */ - - pIn1 = &aMem[pOp->p1]; - applyNumericAffinity(pIn1); - pIn2 = &aMem[pOp->p2]; - applyNumericAffinity(pIn2); - pOut = &aMem[pOp->p3]; - u.af.flags = pIn1->flags | pIn2->flags; - if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; - if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ - u.af.iA = pIn1->u.i; - u.af.iB = pIn2->u.i; - switch( pOp->opcode ){ - case OP_Add: u.af.iB += u.af.iA; break; - case OP_Subtract: u.af.iB -= u.af.iA; break; - case OP_Multiply: u.af.iB *= u.af.iA; break; - case OP_Divide: { - if( u.af.iA==0 ) goto arithmetic_result_is_null; - /* Dividing the largest possible negative 64-bit integer (1<<63) by - ** -1 returns an integer too large to store in a 64-bit data-type. On - ** some architectures, the value overflows to (1<<63). On others, - ** a SIGFPE is issued. The following statement normalizes this - ** behavior so that all architectures behave as if integer - ** overflow occurred. - */ - if( u.af.iA==-1 && u.af.iB==SMALLEST_INT64 ) u.af.iA = 1; - u.af.iB /= u.af.iA; - break; - } - default: { - if( u.af.iA==0 ) goto arithmetic_result_is_null; - if( u.af.iA==-1 ) u.af.iA = 1; - u.af.iB %= u.af.iA; - break; - } +SQLITE_API int sqlite3_bind_text( + sqlite3_stmt *pStmt, + int i, + const char *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_bind_text16( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ + int rc; + switch( pValue->type ){ + case SQLITE_INTEGER: { + rc = sqlite3_bind_int64(pStmt, i, pValue->u.i); + break; } - pOut->u.i = u.af.iB; - MemSetTypeFlag(pOut, MEM_Int); - }else{ - u.af.rA = sqlite3VdbeRealValue(pIn1); - u.af.rB = sqlite3VdbeRealValue(pIn2); - switch( pOp->opcode ){ - case OP_Add: u.af.rB += u.af.rA; break; - case OP_Subtract: u.af.rB -= u.af.rA; break; - case OP_Multiply: u.af.rB *= u.af.rA; break; - case OP_Divide: { - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - if( u.af.rA==(double)0 ) goto arithmetic_result_is_null; - u.af.rB /= u.af.rA; - break; - } - default: { - u.af.iA = (i64)u.af.rA; - u.af.iB = (i64)u.af.rB; - if( u.af.iA==0 ) goto arithmetic_result_is_null; - if( u.af.iA==-1 ) u.af.iA = 1; - u.af.rB = (double)(u.af.iB % u.af.iA); - break; + case SQLITE_FLOAT: { + rc = sqlite3_bind_double(pStmt, i, pValue->r); + break; + } + case SQLITE_BLOB: { + if( pValue->flags & MEM_Zero ){ + rc = sqlite3_bind_zeroblob(pStmt, i, pValue->u.nZero); + }else{ + rc = sqlite3_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE_TRANSIENT); } + break; } -#ifdef SQLITE_OMIT_FLOATING_POINT - pOut->u.i = u.af.rB; - MemSetTypeFlag(pOut, MEM_Int); -#else - if( sqlite3IsNaN(u.af.rB) ){ - goto arithmetic_result_is_null; + case SQLITE_TEXT: { + rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE_TRANSIENT, + pValue->enc); + break; } - pOut->r = u.af.rB; - MemSetTypeFlag(pOut, MEM_Real); - if( (u.af.flags & MEM_Real)==0 ){ - sqlite3VdbeIntegerAffinity(pOut); + default: { + rc = sqlite3_bind_null(pStmt, i); + break; } -#endif } - break; - -arithmetic_result_is_null: - sqlite3VdbeMemSetNull(pOut); - break; + return rc; +} +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; } -/* Opcode: CollSeq * * P4 -** -** P4 is a pointer to a CollSeq struct. If the next call to a user function -** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will -** be returned. This is used by the built-in min(), max() and nullif() -** functions. -** -** The interface used by the implementation of the aforementioned functions -** to retrieve the collation sequence set by this opcode is not available -** publicly, only to user functions defined in func.c. +/* +** Return the number of wildcards that can be potentially bound to. +** This routine is added to support DBD::SQLite. */ -case OP_CollSeq: { - assert( pOp->p4type==P4_COLLSEQ ); - break; +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + return p ? p->nVar : 0; } -/* Opcode: Function P1 P2 P3 P4 P5 -** -** Invoke a user function (P4 is a pointer to a Function structure that -** defines the function) with P5 arguments taken from register P2 and -** successors. The result of the function is stored in register P3. -** Register P3 must not be one of the function inputs. -** -** P1 is a 32-bit bitmask indicating whether or not each argument to the -** function was determined to be constant at compile time. If the first -** argument was constant then bit 0 of P1 is set. This is used to determine -** whether meta data associated with a user function argument using the -** sqlite3_set_auxdata() API may be safely retained until the next -** invocation of this opcode. +/* +** Return the name of a wildcard parameter. Return NULL if the index +** is out of range or if the wildcard is unnamed. ** -** See also: AggStep and AggFinal +** The result is always UTF-8. */ -case OP_Function: { -#if 0 /* local variables moved into u.ag */ - int i; - Mem *pArg; - sqlite3_context ctx; - sqlite3_value **apVal; - int n; -#endif /* local variables moved into u.ag */ - - u.ag.n = pOp->p5; - u.ag.apVal = p->apArg; - assert( u.ag.apVal || u.ag.n==0 ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pOut = &aMem[pOp->p3]; - memAboutToChange(p, pOut); - - assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+u.ag.n ); - u.ag.pArg = &aMem[pOp->p2]; - for(u.ag.i=0; u.ag.ip2+u.ag.i, u.ag.pArg); - } - - assert( pOp->p4type==P4_FUNCDEF || pOp->p4type==P4_VDBEFUNC ); - if( pOp->p4type==P4_FUNCDEF ){ - u.ag.ctx.pFunc = pOp->p4.pFunc; - u.ag.ctx.pVdbeFunc = 0; - }else{ - u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc; - u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc; - } - - u.ag.ctx.s.flags = MEM_Null; - u.ag.ctx.s.db = db; - u.ag.ctx.s.xDel = 0; - u.ag.ctx.s.zMalloc = 0; - - /* The output cell may already have a buffer allocated. Move - ** the pointer to u.ag.ctx.s so in case the user-function can use - ** the already allocated buffer instead of allocating a new one. - */ - sqlite3VdbeMemMove(&u.ag.ctx.s, pOut); - MemSetTypeFlag(&u.ag.ctx.s, MEM_Null); - - u.ag.ctx.isError = 0; - if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - u.ag.ctx.pColl = pOp[-1].p4.pColl; - } - (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&u.ag.ctx.s); - goto no_mem; +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ + Vdbe *p = (Vdbe*)pStmt; + if( p==0 || i<1 || i>p->nzVar ){ + return 0; } + return p->azVar[i-1]; +} - /* If any auxiliary data functions have been called by this user function, - ** immediately call the destructor for any non-static values. - */ - if( u.ag.ctx.pVdbeFunc ){ - sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1); - pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc; - pOp->p4type = P4_VDBEFUNC; +/* +** Given a wildcard parameter name, return the index of the variable +** with that name. If there is no variable with the given name, +** return 0. +*/ +SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ + int i; + if( p==0 ){ + return 0; } - - /* If the function returned an error, throw an exception */ - if( u.ag.ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s)); - rc = u.ag.ctx.isError; + if( zName ){ + for(i=0; inzVar; i++){ + const char *z = p->azVar[i]; + if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ + return i+1; + } + } } + return 0; +} +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ + return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); +} - /* Copy the result of the function into register P3 */ - sqlite3VdbeChangeEncoding(&u.ag.ctx.s, encoding); - sqlite3VdbeMemMove(pOut, &u.ag.ctx.s); - if( sqlite3VdbeMemTooBig(pOut) ){ - goto too_big; +/* +** Transfer all bindings from the first statement over to the second. +*/ +SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ + Vdbe *pFrom = (Vdbe*)pFromStmt; + Vdbe *pTo = (Vdbe*)pToStmt; + int i; + assert( pTo->db==pFrom->db ); + assert( pTo->nVar==pFrom->nVar ); + sqlite3_mutex_enter(pTo->db->mutex); + for(i=0; inVar; i++){ + sqlite3VdbeMemMove(&pTo->aVar[i], &pFrom->aVar[i]); } - REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); - break; + sqlite3_mutex_leave(pTo->db->mutex); + return SQLITE_OK; } -/* Opcode: BitAnd P1 P2 P3 * * -** -** Take the bit-wise AND of the values in register P1 and P2 and -** store the result in register P3. -** If either input is NULL, the result is NULL. -*/ -/* Opcode: BitOr P1 P2 P3 * * -** -** Take the bit-wise OR of the values in register P1 and P2 and -** store the result in register P3. -** If either input is NULL, the result is NULL. -*/ -/* Opcode: ShiftLeft P1 P2 P3 * * +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3TransferBindings. ** -** Shift the integer value in register P2 to the left by the -** number of bits specified by the integer in register P1. -** Store the result in register P3. -** If either input is NULL, the result is NULL. -*/ -/* Opcode: ShiftRight P1 P2 P3 * * +** Is is misuse to call this routine with statements from different +** database connections. But as this is a deprecated interface, we +** will not bother to check for that condition. ** -** Shift the integer value in register P2 to the right by the -** number of bits specified by the integer in register P1. -** Store the result in register P3. -** If either input is NULL, the result is NULL. +** If the two statements contain a different number of bindings, then +** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise +** SQLITE_OK is returned. */ -case OP_BitAnd: /* same as TK_BITAND, in1, in2, out3 */ -case OP_BitOr: /* same as TK_BITOR, in1, in2, out3 */ -case OP_ShiftLeft: /* same as TK_LSHIFT, in1, in2, out3 */ -case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ -#if 0 /* local variables moved into u.ah */ - i64 a; - i64 b; -#endif /* local variables moved into u.ah */ - - pIn1 = &aMem[pOp->p1]; - pIn2 = &aMem[pOp->p2]; - pOut = &aMem[pOp->p3]; - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - break; +SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ + Vdbe *pFrom = (Vdbe*)pFromStmt; + Vdbe *pTo = (Vdbe*)pToStmt; + if( pFrom->nVar!=pTo->nVar ){ + return SQLITE_ERROR; } - u.ah.a = sqlite3VdbeIntValue(pIn2); - u.ah.b = sqlite3VdbeIntValue(pIn1); - switch( pOp->opcode ){ - case OP_BitAnd: u.ah.a &= u.ah.b; break; - case OP_BitOr: u.ah.a |= u.ah.b; break; - case OP_ShiftLeft: u.ah.a <<= u.ah.b; break; - default: assert( pOp->opcode==OP_ShiftRight ); - u.ah.a >>= u.ah.b; break; + if( pTo->isPrepareV2 && pTo->expmask ){ + pTo->expired = 1; } - pOut->u.i = u.ah.a; - MemSetTypeFlag(pOut, MEM_Int); - break; + if( pFrom->isPrepareV2 && pFrom->expmask ){ + pFrom->expired = 1; + } + return sqlite3TransferBindings(pFromStmt, pToStmt); } +#endif -/* Opcode: AddImm P1 P2 * * * -** -** Add the constant P2 to the value in register P1. -** The result is always an integer. -** -** To force any register to be an integer, just add 0. +/* +** Return the sqlite3* database handle to which the prepared statement given +** in the argument belongs. This is the same database handle that was +** the first argument to the sqlite3_prepare() that was used to create +** the statement in the first place. */ -case OP_AddImm: { /* in1 */ - pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); - sqlite3VdbeMemIntegerify(pIn1); - pIn1->u.i += pOp->p2; - break; +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->db : 0; } -/* Opcode: MustBeInt P1 P2 * * * -** -** Force the value in register P1 to be an integer. If the value -** in P1 is not an integer and cannot be converted into an integer -** without data loss, then jump immediately to P2, or if P2==0 -** raise an SQLITE_MISMATCH exception. +/* +** Return true if the prepared statement is guaranteed to not modify the +** database. */ -case OP_MustBeInt: { /* jump, in1 */ - pIn1 = &aMem[pOp->p1]; - applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); - if( (pIn1->flags & MEM_Int)==0 ){ - if( pOp->p2==0 ){ - rc = SQLITE_MISMATCH; - goto abort_due_to_error; - }else{ - pc = pOp->p2 - 1; - } - }else{ - MemSetTypeFlag(pIn1, MEM_Int); - } - break; +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; } -#ifndef SQLITE_OMIT_FLOATING_POINT -/* Opcode: RealAffinity P1 * * * * -** -** If register P1 holds an integer convert it to a real value. -** -** This opcode is used when extracting information from a column that -** has REAL affinity. Such column values may still be stored as -** integers, for space efficiency, but after extraction we want them -** to have only a real value. +/* +** Return a pointer to the next prepared statement after pStmt associated +** with database connection pDb. If pStmt is NULL, return the first +** prepared statement for the database connection. Return NULL if there +** are no more. */ -case OP_RealAffinity: { /* in1 */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Int ){ - sqlite3VdbeMemRealify(pIn1); +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ + sqlite3_stmt *pNext; + sqlite3_mutex_enter(pDb->mutex); + if( pStmt==0 ){ + pNext = (sqlite3_stmt*)pDb->pVdbe; + }else{ + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; } - break; + sqlite3_mutex_leave(pDb->mutex); + return pNext; } -#endif -#ifndef SQLITE_OMIT_CAST -/* Opcode: ToText P1 * * * * -** -** Force the value in register P1 to be text. -** If the value is numeric, convert it to a string using the -** equivalent of printf(). Blob values are unchanged and -** are afterwards simply interpreted as text. -** -** A NULL value is not changed by this routine. It remains NULL. +/* +** Return the value of a status counter for a prepared statement */ -case OP_ToText: { /* same as TK_TO_TEXT, in1 */ - pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); - if( pIn1->flags & MEM_Null ) break; - assert( MEM_Str==(MEM_Blob>>3) ); - pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); - rc = ExpandBlob(pIn1); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); - UPDATE_MAX_BLOBSIZE(pIn1); - break; +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ + Vdbe *pVdbe = (Vdbe*)pStmt; + int v = pVdbe->aCounter[op-1]; + if( resetFlag ) pVdbe->aCounter[op-1] = 0; + return v; } -/* Opcode: ToBlob P1 * * * * +/************** End of vdbeapi.c *********************************************/ +/************** Begin file vdbetrace.c ***************************************/ +/* +** 2009 November 25 ** -** Force the value in register P1 to be a BLOB. -** If the value is numeric, convert it to a string first. -** Strings are simply reinterpreted as blobs with no change -** to the underlying data. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ) break; - if( (pIn1->flags & MEM_Blob)==0 ){ - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - MemSetTypeFlag(pIn1, MEM_Blob); - }else{ - pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob); - } - UPDATE_MAX_BLOBSIZE(pIn1); - break; -} - -/* Opcode: ToNumeric P1 * * * * +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** Force the value in register P1 to be numeric (either an -** integer or a floating-point number.) -** If the value is text or blob, try to convert it to an using the -** equivalent of atoi() or atof() and store 0 if no such conversion -** is possible. +************************************************************************* ** -** A NULL value is not changed by this routine. It remains NULL. +** This file contains code used to insert the values of host parameters +** (aka "wildcards") into the SQL text output by sqlite3_trace(). */ -case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ - pIn1 = &aMem[pOp->p1]; - sqlite3VdbeMemNumerify(pIn1); - break; -} -#endif /* SQLITE_OMIT_CAST */ -/* Opcode: ToInt P1 * * * * -** -** Force the value in register P1 to be an integer. If -** The value is currently a real number, drop its fractional part. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0 if no such conversion is possible. -** -** A NULL value is not changed by this routine. It remains NULL. +#ifndef SQLITE_OMIT_TRACE + +/* +** zSql is a zero-terminated string of UTF-8 SQL text. Return the number of +** bytes in this text up to but excluding the first character in +** a host parameter. If the text contains no host parameters, return +** the total number of bytes in the text. */ -case OP_ToInt: { /* same as TK_TO_INT, in1 */ - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemIntegerify(pIn1); +static int findNextHostParameter(const char *zSql, int *pnToken){ + int tokenType; + int nTotal = 0; + int n; + + *pnToken = 0; + while( zSql[0] ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + assert( n>0 && tokenType!=TK_ILLEGAL ); + if( tokenType==TK_VARIABLE ){ + *pnToken = n; + break; + } + nTotal += n; + zSql += n; } - break; + return nTotal; } -#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) -/* Opcode: ToReal P1 * * * * +/* +** This function returns a pointer to a nul-terminated string in memory +** obtained from sqlite3DbMalloc(). If sqlite3.vdbeExecCnt is 1, then the +** string contains a copy of zRawSql but with host parameters expanded to +** their current bindings. Or, if sqlite3.vdbeExecCnt is greater than 1, +** then the returned string holds a copy of zRawSql with "-- " prepended +** to each line of text. ** -** Force the value in register P1 to be a floating point number. -** If The value is currently an integer, convert it. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0.0 if no such conversion is possible. +** The calling function is responsible for making sure the memory returned +** is eventually freed. ** -** A NULL value is not changed by this routine. It remains NULL. +** ALGORITHM: Scan the input string looking for host parameters in any of +** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within +** string literals, quoted identifier names, and comments. For text forms, +** the host parameter index is found by scanning the perpared +** statement for the corresponding OP_Variable opcode. Once the host +** parameter index is known, locate the value in p->aVar[]. Then render +** the value as a literal in place of the host parameter name. */ -case OP_ToReal: { /* same as TK_TO_REAL, in1 */ - pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemRealify(pIn1); +SQLITE_PRIVATE char *sqlite3VdbeExpandSql( + Vdbe *p, /* The prepared statement being evaluated */ + const char *zRawSql /* Raw text of the SQL statement */ +){ + sqlite3 *db; /* The database connection */ + int idx = 0; /* Index of a host parameter */ + int nextIndex = 1; /* Index of next ? host parameter */ + int n; /* Length of a token prefix */ + int nToken; /* Length of the parameter token */ + int i; /* Loop counter */ + Mem *pVar; /* Value of a host parameter */ + StrAccum out; /* Accumulate the output here */ + char zBase[100]; /* Initial working space */ + + db = p->db; + sqlite3StrAccumInit(&out, zBase, sizeof(zBase), + db->aLimit[SQLITE_LIMIT_LENGTH]); + out.db = db; + if( db->vdbeExecCnt>1 ){ + while( *zRawSql ){ + const char *zStart = zRawSql; + while( *(zRawSql++)!='\n' && *zRawSql ); + sqlite3StrAccumAppend(&out, "-- ", 3); + sqlite3StrAccumAppend(&out, zStart, (int)(zRawSql-zStart)); + } + }else{ + while( zRawSql[0] ){ + n = findNextHostParameter(zRawSql, &nToken); + assert( n>0 ); + sqlite3StrAccumAppend(&out, zRawSql, n); + zRawSql += n; + assert( zRawSql[0] || nToken==0 ); + if( nToken==0 ) break; + if( zRawSql[0]=='?' ){ + if( nToken>1 ){ + assert( sqlite3Isdigit(zRawSql[1]) ); + sqlite3GetInt32(&zRawSql[1], &idx); + }else{ + idx = nextIndex; + } + }else{ + assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' ); + testcase( zRawSql[0]==':' ); + testcase( zRawSql[0]=='$' ); + testcase( zRawSql[0]=='@' ); + idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); + assert( idx>0 ); + } + zRawSql += nToken; + nextIndex = idx + 1; + assert( idx>0 && idx<=p->nVar ); + pVar = &p->aVar[idx-1]; + if( pVar->flags & MEM_Null ){ + sqlite3StrAccumAppend(&out, "NULL", 4); + }else if( pVar->flags & MEM_Int ){ + sqlite3XPrintf(&out, "%lld", pVar->u.i); + }else if( pVar->flags & MEM_Real ){ + sqlite3XPrintf(&out, "%!.15g", pVar->r); + }else if( pVar->flags & MEM_Str ){ +#ifndef SQLITE_OMIT_UTF16 + u8 enc = ENC(db); + if( enc!=SQLITE_UTF8 ){ + Mem utf8; + memset(&utf8, 0, sizeof(utf8)); + utf8.db = db; + sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); + sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); + sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); + sqlite3VdbeMemRelease(&utf8); + }else +#endif + { + sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + } + }else if( pVar->flags & MEM_Zero ){ + sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); + }else{ + assert( pVar->flags & MEM_Blob ); + sqlite3StrAccumAppend(&out, "x'", 2); + for(i=0; in; i++){ + sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); + } + sqlite3StrAccumAppend(&out, "'", 1); + } + } } - break; + return sqlite3StrAccumFinish(&out); } -#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ -/* Opcode: Lt P1 P2 P3 P4 P5 -** -** Compare the values in register P1 and P3. If reg(P3)flags */ - u16 flags3; /* Copy of initial value of pIn3->flags */ -#endif /* local variables moved into u.ai */ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_interrupt_count = 0; +#endif - pIn1 = &aMem[pOp->p1]; - pIn3 = &aMem[pOp->p3]; - u.ai.flags1 = pIn1->flags; - u.ai.flags3 = pIn3->flags; - if( (pIn1->flags | pIn3->flags)&MEM_Null ){ - /* One or both operands are NULL */ - if( pOp->p5 & SQLITE_NULLEQ ){ - /* If SQLITE_NULLEQ is set (which will only happen if the operator is - ** OP_Eq or OP_Ne) then take the jump or not depending on whether - ** or not both operands are null. - */ - assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); - u.ai.res = (pIn1->flags & pIn3->flags & MEM_Null)==0; - }else{ - /* SQLITE_NULLEQ is clear and at least one operand is NULL, - ** then the result is always NULL. - ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. - */ - if( pOp->p5 & SQLITE_STOREP2 ){ - pOut = &aMem[pOp->p2]; - MemSetTypeFlag(pOut, MEM_Null); - REGISTER_TRACE(pOp->p2, pOut); - }else if( pOp->p5 & SQLITE_JUMPIFNULL ){ - pc = pOp->p2-1; - } - break; - } - }else{ - /* Neither operand is NULL. Do a comparison. */ - u.ai.affinity = pOp->p5 & SQLITE_AFF_MASK; - if( u.ai.affinity ){ - applyAffinity(pIn1, u.ai.affinity, encoding); - applyAffinity(pIn3, u.ai.affinity, encoding); - if( db->mallocFailed ) goto no_mem; - } +/* +** The next global variable is incremented each type the OP_Sort opcode +** is executed. The test procedures use this information to make sure that +** sorting is occurring or not occurring at appropriate times. This variable +** has no function other than to help verify the correct operation of the +** library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_sort_count = 0; +#endif - assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - ExpandBlob(pIn1); - ExpandBlob(pIn3); - u.ai.res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); - } - switch( pOp->opcode ){ - case OP_Eq: u.ai.res = u.ai.res==0; break; - case OP_Ne: u.ai.res = u.ai.res!=0; break; - case OP_Lt: u.ai.res = u.ai.res<0; break; - case OP_Le: u.ai.res = u.ai.res<=0; break; - case OP_Gt: u.ai.res = u.ai.res>0; break; - default: u.ai.res = u.ai.res>=0; break; +/* +** The next global variable records the size of the largest MEM_Blob +** or MEM_Str that has been used by a VDBE opcode. The test procedures +** use this information to make sure that the zero-blob functionality +** is working correctly. This variable has no function other than to +** help verify the correct operation of the library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_max_blobsize = 0; +static void updateMaxBlobsize(Mem *p){ + if( (p->flags & (MEM_Str|MEM_Blob))!=0 && p->n>sqlite3_max_blobsize ){ + sqlite3_max_blobsize = p->n; } +} +#endif - if( pOp->p5 & SQLITE_STOREP2 ){ - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); - MemSetTypeFlag(pOut, MEM_Int); - pOut->u.i = u.ai.res; - REGISTER_TRACE(pOp->p2, pOut); - }else if( u.ai.res ){ - pc = pOp->p2-1; - } +/* +** The next global variable is incremented each type the OP_Found opcode +** is executed. This is used to test whether or not the foreign key +** operation implemented using OP_FkIsZero is working. This variable +** has no function other than to help verify the correct operation of the +** library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_found_count = 0; +#endif - /* Undo any changes made by applyAffinity() to the input registers. */ - pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (u.ai.flags1&MEM_TypeMask); - pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (u.ai.flags3&MEM_TypeMask); - break; -} +/* +** Test a register to see if it exceeds the current maximum blob size. +** If it does, record the new maximum blob size. +*/ +#if defined(SQLITE_TEST) && !defined(SQLITE_OMIT_BUILTIN_TEST) +# define UPDATE_MAX_BLOBSIZE(P) updateMaxBlobsize(P) +#else +# define UPDATE_MAX_BLOBSIZE(P) +#endif -/* Opcode: Permutation * * * P4 * -** -** Set the permutation used by the OP_Compare operator to be the array -** of integers in P4. -** -** The permutation is only valid until the next OP_Permutation, OP_Compare, -** OP_Halt, or OP_ResultRow. Typically the OP_Permutation should occur -** immediately prior to the OP_Compare. +/* +** Convert the given register into a string if it isn't one +** already. Return non-zero if a malloc() fails. */ -case OP_Permutation: { - assert( pOp->p4type==P4_INTARRAY ); - assert( pOp->p4.ai ); - aPermute = pOp->p4.ai; - break; -} +#define Stringify(P, enc) \ + if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc)) \ + { goto no_mem; } -/* Opcode: Compare P1 P2 P3 P4 * -** -** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this -** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of -** the comparison for use by the next OP_Jump instruct. -** -** P4 is a KeyInfo structure that defines collating sequences and sort -** orders for the comparison. The permutation applies to registers -** only. The KeyInfo elements are used sequentially. +/* +** An ephemeral string value (signified by the MEM_Ephem flag) contains +** a pointer to a dynamically allocated string where some other entity +** is responsible for deallocating that string. Because the register +** does not control the string, it might be deleted without the register +** knowing it. ** -** The comparison is a sort comparison, so NULLs compare equal, -** NULLs are less than numbers, numbers are less than strings, -** and strings are less than blobs. +** This routine converts an ephemeral string into a dynamically allocated +** string that the register itself controls. In other words, it +** converts an MEM_Ephem string into an MEM_Dyn string. */ -case OP_Compare: { -#if 0 /* local variables moved into u.aj */ - int n; - int i; - int p1; - int p2; - const KeyInfo *pKeyInfo; - int idx; - CollSeq *pColl; /* Collating sequence to use on this term */ - int bRev; /* True for DESCENDING sort order */ -#endif /* local variables moved into u.aj */ +#define Deephemeralize(P) \ + if( ((P)->flags&MEM_Ephem)!=0 \ + && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} - u.aj.n = pOp->p3; - u.aj.pKeyInfo = pOp->p4.pKeyInfo; - assert( u.aj.n>0 ); - assert( u.aj.pKeyInfo!=0 ); - u.aj.p1 = pOp->p1; - u.aj.p2 = pOp->p2; -#if SQLITE_DEBUG - if( aPermute ){ - int k, mx = 0; - for(k=0; kmx ) mx = aPermute[k]; - assert( u.aj.p1>0 && u.aj.p1+mx<=p->nMem+1 ); - assert( u.aj.p2>0 && u.aj.p2+mx<=p->nMem+1 ); - }else{ - assert( u.aj.p1>0 && u.aj.p1+u.aj.n<=p->nMem+1 ); - assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 ); - } -#endif /* SQLITE_DEBUG */ - for(u.aj.i=0; u.aj.inField ); - u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i]; - u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i]; - iCompare = sqlite3MemCompare(&aMem[u.aj.p1+u.aj.idx], &aMem[u.aj.p2+u.aj.idx], u.aj.pColl); - if( iCompare ){ - if( u.aj.bRev ) iCompare = -iCompare; - break; - } - } - aPermute = 0; - break; -} - -/* Opcode: Jump P1 P2 P3 * * -** -** Jump to the instruction at address P1, P2, or P3 depending on whether -** in the most recent OP_Compare instruction the P1 vector was less than -** equal to, or greater than the P2 vector, respectively. +/* +** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*) +** P if required. */ -case OP_Jump: { /* jump */ - if( iCompare<0 ){ - pc = pOp->p1 - 1; - }else if( iCompare==0 ){ - pc = pOp->p2 - 1; - }else{ - pc = pOp->p3 - 1; - } - break; -} +#define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) -/* Opcode: And P1 P2 P3 * * -** -** Take the logical AND of the values in registers P1 and P2 and -** write the result into register P3. -** -** If either P1 or P2 is 0 (false) then the result is 0 even if -** the other input is NULL. A NULL and true or two NULLs give -** a NULL output. -*/ -/* Opcode: Or P1 P2 P3 * * -** -** Take the logical OR of the values in register P1 and P2 and -** store the answer in register P3. -** -** If either P1 or P2 is nonzero (true) then the result is 1 (true) -** even if the other input is NULL. A NULL and false or two NULLs -** give a NULL output. +/* +** Argument pMem points at a register that will be passed to a +** user-defined function or returned to the user as the result of a query. +** This routine sets the pMem->type variable used by the sqlite3_value_*() +** routines. */ -case OP_And: /* same as TK_AND, in1, in2, out3 */ -case OP_Or: { /* same as TK_OR, in1, in2, out3 */ -#if 0 /* local variables moved into u.ak */ - int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ - int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ -#endif /* local variables moved into u.ak */ - - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - u.ak.v1 = 2; - }else{ - u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0; +SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem){ + int flags = pMem->flags; + if( flags & MEM_Null ){ + pMem->type = SQLITE_NULL; } - pIn2 = &aMem[pOp->p2]; - if( pIn2->flags & MEM_Null ){ - u.ak.v2 = 2; - }else{ - u.ak.v2 = sqlite3VdbeIntValue(pIn2)!=0; + else if( flags & MEM_Int ){ + pMem->type = SQLITE_INTEGER; } - if( pOp->opcode==OP_And ){ - static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; - u.ak.v1 = and_logic[u.ak.v1*3+u.ak.v2]; - }else{ - static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 }; - u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2]; + else if( flags & MEM_Real ){ + pMem->type = SQLITE_FLOAT; } - pOut = &aMem[pOp->p3]; - if( u.ak.v1==2 ){ - MemSetTypeFlag(pOut, MEM_Null); + else if( flags & MEM_Str ){ + pMem->type = SQLITE_TEXT; }else{ - pOut->u.i = u.ak.v1; - MemSetTypeFlag(pOut, MEM_Int); + pMem->type = SQLITE_BLOB; } - break; } -/* Opcode: Not P1 P2 * * * -** -** Interpret the value in register P1 as a boolean value. Store the -** boolean complement in register P2. If the value in register P1 is -** NULL, then a NULL is stored in P2. +/* +** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL +** if we run out of memory. */ -case OP_Not: { /* same as TK_NOT, in1, out2 */ - pIn1 = &aMem[pOp->p1]; - pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); +static VdbeCursor *allocateCursor( + Vdbe *p, /* The virtual machine */ + int iCur, /* Index of the new VdbeCursor */ + int nField, /* Number of fields in the table or index */ + int iDb, /* When database the cursor belongs to, or -1 */ + int isBtreeCursor /* True for B-Tree. False for pseudo-table or vtab */ +){ + /* Find the memory cell that will be used to store the blob of memory + ** required for this VdbeCursor structure. It is convenient to use a + ** vdbe memory cell to manage the memory allocation required for a + ** VdbeCursor structure for the following reasons: + ** + ** * Sometimes cursor numbers are used for a couple of different + ** purposes in a vdbe program. The different uses might require + ** different sized allocations. Memory cells provide growable + ** allocations. + ** + ** * When using ENABLE_MEMORY_MANAGEMENT, memory cell buffers can + ** be freed lazily via the sqlite3_release_memory() API. This + ** minimizes the number of malloc calls made by the system. + ** + ** Memory cells for cursors are allocated at the top of the address + ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for + ** cursor 1 is managed by memory cell (p->nMem-1), etc. + */ + Mem *pMem = &p->aMem[p->nMem-iCur]; + + int nByte; + VdbeCursor *pCx = 0; + nByte = + ROUND8(sizeof(VdbeCursor)) + + (isBtreeCursor?sqlite3BtreeCursorSize():0) + + 2*nField*sizeof(u32); + + assert( iCurnCursor ); + if( p->apCsr[iCur] ){ + sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); + p->apCsr[iCur] = 0; } - break; + if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; + memset(pCx, 0, sizeof(VdbeCursor)); + pCx->iDb = iDb; + pCx->nField = nField; + if( nField ){ + pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))]; + } + if( isBtreeCursor ){ + pCx->pCursor = (BtCursor*) + &pMem->z[ROUND8(sizeof(VdbeCursor))+2*nField*sizeof(u32)]; + sqlite3BtreeCursorZero(pCx->pCursor); + } + } + return pCx; } -/* Opcode: BitNot P1 P2 * * * -** -** Interpret the content of register P1 as an integer. Store the -** ones-complement of the P1 value into register P2. If P1 holds -** a NULL then store a NULL in P2. +/* +** Try to convert a value into a numeric representation if we can +** do so without loss of information. In other words, if the string +** looks like a number, convert it into a number. If it does not +** look like a number, leave it alone. */ -case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ - pIn1 = &aMem[pOp->p1]; - pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1)); +static void applyNumericAffinity(Mem *pRec){ + if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ + double rValue; + i64 iValue; + u8 enc = pRec->enc; + if( (pRec->flags&MEM_Str)==0 ) return; + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->r = rValue; + pRec->flags |= MEM_Real; + } } - break; } -/* Opcode: If P1 P2 P3 * * +/* +** Processing is determine by the affinity parameter: ** -** Jump to P2 if the value in register P1 is true. The value is -** is considered true if it is numeric and non-zero. If the value -** in P1 is NULL then take the jump if P3 is true. -*/ -/* Opcode: IfNot P1 P2 P3 * * +** SQLITE_AFF_INTEGER: +** SQLITE_AFF_REAL: +** SQLITE_AFF_NUMERIC: +** Try to convert pRec to an integer representation or a +** floating-point representation if an integer representation +** is not possible. Note that the integer representation is +** always preferred, even if the affinity is REAL, because +** an integer representation is more space efficient on disk. ** -** Jump to P2 if the value in register P1 is False. The value is -** is considered true if it has a numeric value of zero. If the value -** in P1 is NULL then take the jump if P3 is true. +** SQLITE_AFF_TEXT: +** Convert pRec to a text representation. +** +** SQLITE_AFF_NONE: +** No-op. pRec is unchanged. */ -case OP_If: /* jump, in1 */ -case OP_IfNot: { /* jump, in1 */ -#if 0 /* local variables moved into u.al */ - int c; -#endif /* local variables moved into u.al */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - u.al.c = pOp->p3; - }else{ -#ifdef SQLITE_OMIT_FLOATING_POINT - u.al.c = sqlite3VdbeIntValue(pIn1)!=0; -#else - u.al.c = sqlite3VdbeRealValue(pIn1)!=0.0; -#endif - if( pOp->opcode==OP_IfNot ) u.al.c = !u.al.c; - } - if( u.al.c ){ - pc = pOp->p2-1; +static void applyAffinity( + Mem *pRec, /* The value to apply affinity to */ + char affinity, /* The affinity to be applied */ + u8 enc /* Use this text encoding */ +){ + if( affinity==SQLITE_AFF_TEXT ){ + /* Only attempt the conversion to TEXT if there is an integer or real + ** representation (blob and NULL do not get converted) but no string + ** representation. + */ + if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc); + } + pRec->flags &= ~(MEM_Real|MEM_Int); + }else if( affinity!=SQLITE_AFF_NONE ){ + assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL + || affinity==SQLITE_AFF_NUMERIC ); + applyNumericAffinity(pRec); + if( pRec->flags & MEM_Real ){ + sqlite3VdbeIntegerAffinity(pRec); + } } - break; } -/* Opcode: IsNull P1 P2 * * * -** -** Jump to P2 if the value in register P1 is NULL. +/* +** Try to convert the type of a function argument or a result column +** into a numeric representation. Use either INTEGER or REAL whichever +** is appropriate. But only do the conversion if it is possible without +** loss of information and return the revised type of the argument. */ -case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_Null)!=0 ){ - pc = pOp->p2 - 1; +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ + Mem *pMem = (Mem*)pVal; + if( pMem->type==SQLITE_TEXT ){ + applyNumericAffinity(pMem); + sqlite3VdbeMemStoreType(pMem); } - break; + return pMem->type; } -/* Opcode: NotNull P1 P2 * * * -** -** Jump to P2 if the value in register P1 is not NULL. +/* +** Exported version of applyAffinity(). This one works on sqlite3_value*, +** not the internal Mem* type. */ -case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_Null)==0 ){ - pc = pOp->p2 - 1; - } - break; +SQLITE_PRIVATE void sqlite3ValueApplyAffinity( + sqlite3_value *pVal, + u8 affinity, + u8 enc +){ + applyAffinity((Mem *)pVal, affinity, enc); } -/* Opcode: Column P1 P2 P3 P4 P5 -** -** Interpret the data that cursor P1 points to as a structure built using -** the MakeRecord instruction. (See the MakeRecord opcode for additional -** information about the format of the data.) Extract the P2-th column -** from this record. If there are less that (P2+1) -** values in the record, extract a NULL. -** -** The value extracted is stored in register P3. -** -** If the column contains fewer than P2 fields, then extract a NULL. Or, -** if the P4 argument is a P4_MEM use the value of the P4 argument as -** the result. -** -** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor, -** then the cache of the cursor is reset prior to extracting the column. -** The first OP_Column against a pseudo-table after the value of the content -** register has changed should have this bit set. +#ifdef SQLITE_DEBUG +/* +** Write a nice string representation of the contents of cell pMem +** into buffer zBuf, length nBuf. */ -case OP_Column: { -#if 0 /* local variables moved into u.am */ - u32 payloadSize; /* Number of bytes in the record */ - i64 payloadSize64; /* Number of bytes in the record */ - int p1; /* P1 value of the opcode */ - int p2; /* column number to retrieve */ - VdbeCursor *pC; /* The VDBE cursor */ - char *zRec; /* Pointer to complete record-data */ - BtCursor *pCrsr; /* The BTree cursor */ - u32 *aType; /* aType[i] holds the numeric type of the i-th column */ - u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ - int nField; /* number of fields in the record */ - int len; /* The length of the serialized data for the column */ - int i; /* Loop counter */ - char *zData; /* Part of the record being decoded */ - Mem *pDest; /* Where to write the extracted value */ - Mem sMem; /* For storing the record being decoded */ - u8 *zIdx; /* Index into header */ - u8 *zEndHdr; /* Pointer to first byte after the header */ - u32 offset; /* Offset into the data */ - u32 szField; /* Number of bytes in the content of a field */ - int szHdr; /* Size of the header size field at start of record */ - int avail; /* Number of bytes of available data */ - Mem *pReg; /* PseudoTable input register */ -#endif /* local variables moved into u.am */ - +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ + char *zCsr = zBuf; + int f = pMem->flags; - u.am.p1 = pOp->p1; - u.am.p2 = pOp->p2; - u.am.pC = 0; - memset(&u.am.sMem, 0, sizeof(u.am.sMem)); - assert( u.am.p1nCursor ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - u.am.pDest = &aMem[pOp->p3]; - memAboutToChange(p, u.am.pDest); - MemSetTypeFlag(u.am.pDest, MEM_Null); - u.am.zRec = 0; + static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; - /* This block sets the variable u.am.payloadSize to be the total number of - ** bytes in the record. - ** - ** u.am.zRec is set to be the complete text of the record if it is available. - ** The complete record text is always available for pseudo-tables - ** If the record is stored in a cursor, the complete record text - ** might be available in the u.am.pC->aRow cache. Or it might not be. - ** If the data is unavailable, u.am.zRec is set to NULL. - ** - ** We also compute the number of columns in the record. For cursors, - ** the number of columns is stored in the VdbeCursor.nField element. - */ - u.am.pC = p->apCsr[u.am.p1]; - assert( u.am.pC!=0 ); -#ifndef SQLITE_OMIT_VIRTUALTABLE - assert( u.am.pC->pVtabCursor==0 ); -#endif - u.am.pCrsr = u.am.pC->pCursor; - if( u.am.pCrsr!=0 ){ - /* The record is stored in a B-Tree */ - rc = sqlite3VdbeCursorMoveto(u.am.pC); - if( rc ) goto abort_due_to_error; - if( u.am.pC->nullRow ){ - u.am.payloadSize = 0; - }else if( u.am.pC->cacheStatus==p->cacheCtr ){ - u.am.payloadSize = u.am.pC->payloadSize; - u.am.zRec = (char*)u.am.pC->aRow; - }else if( u.am.pC->isIndex ){ - assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) ); - rc = sqlite3BtreeKeySize(u.am.pCrsr, &u.am.payloadSize64); - assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ - /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the - ** payload size, so it is impossible for u.am.payloadSize64 to be - ** larger than 32 bits. */ - assert( (u.am.payloadSize64 & SQLITE_MAX_U32)==(u64)u.am.payloadSize64 ); - u.am.payloadSize = (u32)u.am.payloadSize64; + if( f&MEM_Blob ){ + int i; + char c; + if( f & MEM_Dyn ){ + c = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ + c = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ + c = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); }else{ - assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) ); - rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize); - assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + c = 's'; } - }else if( u.am.pC->pseudoTableReg>0 ){ - u.am.pReg = &aMem[u.am.pC->pseudoTableReg]; - assert( u.am.pReg->flags & MEM_Blob ); - assert( memIsValid(u.am.pReg) ); - u.am.payloadSize = u.am.pReg->n; - u.am.zRec = u.am.pReg->z; - u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr; - assert( u.am.payloadSize==0 || u.am.zRec!=0 ); - }else{ - /* Consider the row to be NULL */ - u.am.payloadSize = 0; - } - - /* If u.am.payloadSize is 0, then just store a NULL */ - if( u.am.payloadSize==0 ){ - assert( u.am.pDest->flags&MEM_Null ); - goto op_column_out; - } - assert( db->aLimit[SQLITE_LIMIT_LENGTH]>=0 ); - if( u.am.payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } - - u.am.nField = u.am.pC->nField; - assert( u.am.p2aType; - if( u.am.pC->cacheStatus==p->cacheCtr ){ - u.am.aOffset = u.am.pC->aOffset; - }else{ - assert(u.am.aType); - u.am.avail = 0; - u.am.pC->aOffset = u.am.aOffset = &u.am.aType[u.am.nField]; - u.am.pC->payloadSize = u.am.payloadSize; - u.am.pC->cacheStatus = p->cacheCtr; + sqlite3_snprintf(100, zCsr, "%c", c); + zCsr += sqlite3Strlen30(zCsr); + sqlite3_snprintf(100, zCsr, "%d[", pMem->n); + zCsr += sqlite3Strlen30(zCsr); + for(i=0; i<16 && in; i++){ + sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); + zCsr += sqlite3Strlen30(zCsr); + } + for(i=0; i<16 && in; i++){ + char z = pMem->z[i]; + if( z<32 || z>126 ) *zCsr++ = '.'; + else *zCsr++ = z; + } - /* Figure out how many bytes are in the header */ - if( u.am.zRec ){ - u.am.zData = u.am.zRec; + sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]); + zCsr += sqlite3Strlen30(zCsr); + if( f & MEM_Zero ){ + sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); + zCsr += sqlite3Strlen30(zCsr); + } + *zCsr = '\0'; + }else if( f & MEM_Str ){ + int j, k; + zBuf[0] = ' '; + if( f & MEM_Dyn ){ + zBuf[1] = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ + zBuf[1] = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ + zBuf[1] = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); }else{ - if( u.am.pC->isIndex ){ - u.am.zData = (char*)sqlite3BtreeKeyFetch(u.am.pCrsr, &u.am.avail); + zBuf[1] = 's'; + } + k = 2; + sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); + k += sqlite3Strlen30(&zBuf[k]); + zBuf[k++] = '['; + for(j=0; j<15 && jn; j++){ + u8 c = pMem->z[j]; + if( c>=0x20 && c<0x7f ){ + zBuf[k++] = c; }else{ - u.am.zData = (char*)sqlite3BtreeDataFetch(u.am.pCrsr, &u.am.avail); + zBuf[k++] = '.'; } - /* If KeyFetch()/DataFetch() managed to get the entire payload, - ** save the payload in the u.am.pC->aRow cache. That will save us from - ** having to make additional calls to fetch the content portion of - ** the record. - */ - assert( u.am.avail>=0 ); - if( u.am.payloadSize <= (u32)u.am.avail ){ - u.am.zRec = u.am.zData; - u.am.pC->aRow = (u8*)u.am.zData; - }else{ - u.am.pC->aRow = 0; - } - } - /* The following assert is true in all cases accept when - ** the database file has been corrupted externally. - ** assert( u.am.zRec!=0 || u.am.avail>=u.am.payloadSize || u.am.avail>=9 ); */ - u.am.szHdr = getVarint32((u8*)u.am.zData, u.am.offset); - - /* Make sure a corrupt database has not given us an oversize header. - ** Do this now to avoid an oversize memory allocation. - ** - ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte - ** types use so much data space that there can only be 4096 and 32 of - ** them, respectively. So the maximum header length results from a - ** 3-byte type for each of the maximum of 32768 columns plus three - ** extra bytes for the header length itself. 32768*3 + 3 = 98307. - */ - if( u.am.offset > 98307 ){ - rc = SQLITE_CORRUPT_BKPT; - goto op_column_out; - } - - /* Compute in u.am.len the number of bytes of data we need to read in order - ** to get u.am.nField type values. u.am.offset is an upper bound on this. But - ** u.am.nField might be significantly less than the true number of columns - ** in the table, and in that case, 5*u.am.nField+3 might be smaller than u.am.offset. - ** We want to minimize u.am.len in order to limit the size of the memory - ** allocation, especially if a corrupt database file has caused u.am.offset - ** to be oversized. Offset is limited to 98307 above. But 98307 might - ** still exceed Robson memory allocation limits on some configurations. - ** On systems that cannot tolerate large memory allocations, u.am.nField*5+3 - ** will likely be much smaller since u.am.nField will likely be less than - ** 20 or so. This insures that Robson memory allocation limits are - ** not exceeded even for corrupt database files. - */ - u.am.len = u.am.nField*5 + 3; - if( u.am.len > (int)u.am.offset ) u.am.len = (int)u.am.offset; - - /* The KeyFetch() or DataFetch() above are fast and will get the entire - ** record header in most cases. But they will fail to get the complete - ** record header if the record header does not fit on a single page - ** in the B-Tree. When that happens, use sqlite3VdbeMemFromBtree() to - ** acquire the complete header text. - */ - if( !u.am.zRec && u.am.availisIndex, &u.am.sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; - } - u.am.zData = u.am.sMem.z; - } - u.am.zEndHdr = (u8 *)&u.am.zData[u.am.len]; - u.am.zIdx = (u8 *)&u.am.zData[u.am.szHdr]; - - /* Scan the header and use it to fill in the u.am.aType[] and u.am.aOffset[] - ** arrays. u.am.aType[u.am.i] will contain the type integer for the u.am.i-th - ** column and u.am.aOffset[u.am.i] will contain the u.am.offset from the beginning - ** of the record to the start of the data for the u.am.i-th column - */ - for(u.am.i=0; u.am.i u.am.zEndHdr) || (u.am.offset > u.am.payloadSize) - || (u.am.zIdx==u.am.zEndHdr && u.am.offset!=u.am.payloadSize) ){ - rc = SQLITE_CORRUPT_BKPT; - goto op_column_out; } + zBuf[k++] = ']'; + sqlite3_snprintf(100,&zBuf[k], encnames[pMem->enc]); + k += sqlite3Strlen30(&zBuf[k]); + zBuf[k++] = 0; } +} +#endif - /* Get the column information. If u.am.aOffset[u.am.p2] is non-zero, then - ** deserialize the value from the record. If u.am.aOffset[u.am.p2] is zero, - ** then there are not enough fields in the record to satisfy the - ** request. In this case, set the value NULL or to P4 if P4 is - ** a pointer to a Mem object. - */ - if( u.am.aOffset[u.am.p2] ){ - assert( rc==SQLITE_OK ); - if( u.am.zRec ){ - sqlite3VdbeMemReleaseExternal(u.am.pDest); - sqlite3VdbeSerialGet((u8 *)&u.am.zRec[u.am.aOffset[u.am.p2]], u.am.aType[u.am.p2], u.am.pDest); - }else{ - u.am.len = sqlite3VdbeSerialTypeLen(u.am.aType[u.am.p2]); - sqlite3VdbeMemMove(&u.am.sMem, u.am.pDest); - rc = sqlite3VdbeMemFromBtree(u.am.pCrsr, u.am.aOffset[u.am.p2], u.am.len, u.am.pC->isIndex, &u.am.sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; - } - u.am.zData = u.am.sMem.z; - sqlite3VdbeSerialGet((u8*)u.am.zData, u.am.aType[u.am.p2], u.am.pDest); - } - u.am.pDest->enc = encoding; +#ifdef SQLITE_DEBUG +/* +** Print the value of a register for tracing purposes: +*/ +static void memTracePrint(FILE *out, Mem *p){ + if( p->flags & MEM_Null ){ + fprintf(out, " NULL"); + }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ + fprintf(out, " si:%lld", p->u.i); + }else if( p->flags & MEM_Int ){ + fprintf(out, " i:%lld", p->u.i); +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( p->flags & MEM_Real ){ + fprintf(out, " r:%g", p->r); +#endif + }else if( p->flags & MEM_RowSet ){ + fprintf(out, " (rowset)"); }else{ - if( pOp->p4type==P4_MEM ){ - sqlite3VdbeMemShallowCopy(u.am.pDest, pOp->p4.pMem, MEM_Static); - }else{ - assert( u.am.pDest->flags&MEM_Null ); - } + char zBuf[200]; + sqlite3VdbeMemPrettyPrint(p, zBuf); + fprintf(out, " "); + fprintf(out, "%s", zBuf); } +} +static void registerTrace(FILE *out, int iReg, Mem *p){ + fprintf(out, "REG[%d] = ", iReg); + memTracePrint(out, p); + fprintf(out, "\n"); +} +#endif - /* If we dynamically allocated space to hold the data (in the - ** sqlite3VdbeMemFromBtree() call above) then transfer control of that - ** dynamically allocated space over to the u.am.pDest structure. - ** This prevents a memory copy. - */ - if( u.am.sMem.zMalloc ){ - assert( u.am.sMem.z==u.am.sMem.zMalloc ); - assert( !(u.am.pDest->flags & MEM_Dyn) ); - assert( !(u.am.pDest->flags & (MEM_Blob|MEM_Str)) || u.am.pDest->z==u.am.sMem.z ); - u.am.pDest->flags &= ~(MEM_Ephem|MEM_Static); - u.am.pDest->flags |= MEM_Term; - u.am.pDest->z = u.am.sMem.z; - u.am.pDest->zMalloc = u.am.sMem.zMalloc; - } +#ifdef SQLITE_DEBUG +# define REGISTER_TRACE(R,M) if(p->trace)registerTrace(p->trace,R,M) +#else +# define REGISTER_TRACE(R,M) +#endif - rc = sqlite3VdbeMemMakeWriteable(u.am.pDest); -op_column_out: - UPDATE_MAX_BLOBSIZE(u.am.pDest); - REGISTER_TRACE(pOp->p3, u.am.pDest); - break; -} +#ifdef VDBE_PROFILE -/* Opcode: Affinity P1 P2 * P4 * -** -** Apply affinities to a range of P2 registers starting with P1. -** -** P4 is a string that is P2 characters long. The nth character of the -** string indicates the column affinity that should be used for the nth -** memory cell in the range. +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. */ -case OP_Affinity: { -#if 0 /* local variables moved into u.an */ - const char *zAffinity; /* The affinity to be applied */ - char cAff; /* A single character of affinity */ -#endif /* local variables moved into u.an */ - - u.an.zAffinity = pOp->p4.z; - assert( u.an.zAffinity!=0 ); - assert( u.an.zAffinity[pOp->p2]==0 ); - pIn1 = &aMem[pOp->p1]; - while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){ - assert( pIn1 <= &p->aMem[p->nMem] ); - assert( memIsValid(pIn1) ); - ExpandBlob(pIn1); - applyAffinity(pIn1, u.an.cAff, encoding); - pIn1++; - } - break; -} - -/* Opcode: MakeRecord P1 P2 P3 P4 * +/************** Include hwtime.h in the middle of vdbe.c *********************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 ** -** Convert P2 registers beginning with P1 into the [record format] -** use as a data record in a database table or as a key -** in an index. The OP_Column opcode can decode the record later. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** P4 may be a string that is P2 characters long. The nth character of the -** string indicates the column affinity that should be used for the nth -** field of the index key. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** The mapping from character to affinity is given by the SQLITE_AFF_ -** macros defined in sqliteInt.h. +****************************************************************************** ** -** If P4 is NULL then all index fields have the affinity NONE. +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. */ -case OP_MakeRecord: { -#if 0 /* local variables moved into u.ao */ - u8 *zNewRecord; /* A buffer to hold the data for the new record */ - Mem *pRec; /* The new record */ - u64 nData; /* Number of bytes of data space */ - int nHdr; /* Number of bytes of header space */ - i64 nByte; /* Data space required for this record */ - int nZero; /* Number of zero bytes at the end of the record */ - int nVarint; /* Number of bytes in a varint */ - u32 serial_type; /* Type field */ - Mem *pData0; /* First field to be combined into the record */ - Mem *pLast; /* Last field of the record */ - int nField; /* Number of fields in the record */ - char *zAffinity; /* The affinity string for the record */ - int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] */ - int len; /* Length of a field */ -#endif /* local variables moved into u.ao */ +#ifndef _HWTIME_H_ +#define _HWTIME_H_ - /* Assuming the record contains N fields, the record format looks - ** like this: - ** - ** ------------------------------------------------------------------------ - ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | - ** ------------------------------------------------------------------------ - ** - ** Data(0) is taken from register P1. Data(1) comes from register P1+1 - ** and so froth. - ** - ** Each type field is a varint representing the serial type of the - ** corresponding data element (see sqlite3VdbeSerialType()). The - ** hdr-size field is also a varint which is the offset from the beginning - ** of the record to data0. - */ - u.ao.nData = 0; /* Number of bytes of data space */ - u.ao.nHdr = 0; /* Number of bytes of header space */ - u.ao.nByte = 0; /* Data space required for this record */ - u.ao.nZero = 0; /* Number of zero bytes at the end of the record */ - u.ao.nField = pOp->p1; - u.ao.zAffinity = pOp->p4.z; - assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 ); - u.ao.pData0 = &aMem[u.ao.nField]; - u.ao.nField = pOp->p2; - u.ao.pLast = &u.ao.pData0[u.ao.nField-1]; - u.ao.file_format = p->minWriteFileFormat; +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) - /* Identify the output register */ - assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); - pOut = &aMem[pOp->p3]; - memAboutToChange(p, pOut); + #if defined(__GNUC__) - /* Loop through the elements that will make up the record to figure - ** out how much space is required for the new record. - */ - for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ - assert( memIsValid(u.ao.pRec) ); - if( u.ao.zAffinity ){ - applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding); - } - if( u.ao.pRec->flags&MEM_Zero && u.ao.pRec->n>0 ){ - sqlite3VdbeMemExpandBlob(u.ao.pRec); - } - u.ao.serial_type = sqlite3VdbeSerialType(u.ao.pRec, u.ao.file_format); - u.ao.len = sqlite3VdbeSerialTypeLen(u.ao.serial_type); - u.ao.nData += u.ao.len; - u.ao.nHdr += sqlite3VarintLen(u.ao.serial_type); - if( u.ao.pRec->flags & MEM_Zero ){ - /* Only pure zero-filled BLOBs can be input to this Opcode. - ** We do not allow blobs with a prefix and a zero-filled tail. */ - u.ao.nZero += u.ao.pRec->u.nZero; - }else if( u.ao.len ){ - u.ao.nZero = 0; - } + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; } - /* Add the initial header varint and total the size */ - u.ao.nHdr += u.ao.nVarint = sqlite3VarintLen(u.ao.nHdr); - if( u.ao.nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } + #elif defined(_MSC_VER) - /* Make sure the output register has a buffer large enough to store - ** the new record. The output register (pOp->p3) is not allowed to - ** be one of the input registers (because the following call to - ** sqlite3VdbeMemGrow() could clobber the value before it is used). - */ - if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){ - goto no_mem; + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } } - u.ao.zNewRecord = (u8 *)pOut->z; - /* Write the record */ - u.ao.i = putVarint32(u.ao.zNewRecord, u.ao.nHdr); - for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ - u.ao.serial_type = sqlite3VdbeSerialType(u.ao.pRec, u.ao.file_format); - u.ao.i += putVarint32(&u.ao.zNewRecord[u.ao.i], u.ao.serial_type); /* serial type */ - } - for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ /* serial data */ - u.ao.i += sqlite3VdbeSerialPut(&u.ao.zNewRecord[u.ao.i], (int)(u.ao.nByte-u.ao.i), u.ao.pRec,u.ao.file_format); + #endif + +#elif (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; } - assert( u.ao.i==u.ao.nByte ); + +#elif (defined(__GNUC__) && defined(__ppc__)) - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pOut->n = (int)u.ao.nByte; - pOut->flags = MEM_Blob | MEM_Dyn; - pOut->xDel = 0; - if( u.ao.nZero ){ - pOut->u.nZero = u.ao.nZero; - pOut->flags |= MEM_Zero; + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; } - pOut->enc = SQLITE_UTF8; /* In case the blob is ever converted to text */ - REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); - break; -} -/* Opcode: Count P1 P2 * * * -** -** Store the number of entries (an integer value) in the table or index -** opened by cursor P1 in register P2 -*/ -#ifndef SQLITE_OMIT_BTREECOUNT -case OP_Count: { /* out2-prerelease */ -#if 0 /* local variables moved into u.ap */ - i64 nEntry; - BtCursor *pCrsr; -#endif /* local variables moved into u.ap */ +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - u.ap.pCrsr = p->apCsr[pOp->p1]->pCursor; - if( u.ap.pCrsr ){ - rc = sqlite3BtreeCount(u.ap.pCrsr, &u.ap.nEntry); - }else{ - u.ap.nEntry = 0; - } - pOut->u.i = u.ap.nEntry; - break; -} #endif -/* Opcode: Savepoint P1 * * P4 * +#endif /* !defined(_HWTIME_H_) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in vdbe.c ***********************/ + +#endif + +/* +** The CHECK_FOR_INTERRUPT macro defined here looks to see if the +** sqlite3_interrupt() routine has been called. If it has been, then +** processing of the VDBE program is interrupted. ** -** Open, release or rollback the savepoint named by parameter P4, depending -** on the value of P1. To open a new savepoint, P1==0. To release (commit) an -** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. +** This macro added to every instruction that does a jump in order to +** implement a loop. This test used to be on every single instruction, +** but that meant we more testing that we needed. By only testing the +** flag on jump instructions, we get a (small) speed improvement. */ -case OP_Savepoint: { -#if 0 /* local variables moved into u.aq */ - int p1; /* Value of P1 operand */ - char *zName; /* Name of savepoint */ - int nName; - Savepoint *pNew; - Savepoint *pSavepoint; - Savepoint *pTmp; - int iSavepoint; - int ii; -#endif /* local variables moved into u.aq */ +#define CHECK_FOR_INTERRUPT \ + if( db->u1.isInterrupted ) goto abort_due_to_interrupt; - u.aq.p1 = pOp->p1; - u.aq.zName = pOp->p4.z; - /* Assert that the u.aq.p1 parameter is valid. Also that if there is no open - ** transaction, then there cannot be any savepoints. - */ - assert( db->pSavepoint==0 || db->autoCommit==0 ); - assert( u.aq.p1==SAVEPOINT_BEGIN||u.aq.p1==SAVEPOINT_RELEASE||u.aq.p1==SAVEPOINT_ROLLBACK ); - assert( db->pSavepoint || db->isTransactionSavepoint==0 ); - assert( checkSavepointCount(db) ); +#ifndef NDEBUG +/* +** This function is only called from within an assert() expression. It +** checks that the sqlite3.nTransaction variable is correctly set to +** the number of non-transaction savepoints currently in the +** linked list starting at sqlite3.pSavepoint. +** +** Usage: +** +** assert( checkSavepointCount(db) ); +*/ +static int checkSavepointCount(sqlite3 *db){ + int n = 0; + Savepoint *p; + for(p=db->pSavepoint; p; p=p->pNext) n++; + assert( n==(db->nSavepoint + db->isTransactionSavepoint) ); + return 1; +} +#endif - if( u.aq.p1==SAVEPOINT_BEGIN ){ - if( db->writeVdbeCnt>0 ){ - /* A new savepoint cannot be created if there are active write - ** statements (i.e. open read/write incremental blob handles). - */ - sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else{ - u.aq.nName = sqlite3Strlen30(u.aq.zName); +/* +** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored +** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored +** in memory obtained from sqlite3DbMalloc). +*/ +static void importVtabErrMsg(Vdbe *p, sqlite3_vtab *pVtab){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; +} - /* Create a new savepoint structure. */ - u.aq.pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+u.aq.nName+1); - if( u.aq.pNew ){ - u.aq.pNew->zName = (char *)&u.aq.pNew[1]; - memcpy(u.aq.pNew->zName, u.aq.zName, u.aq.nName+1); - /* If there is no open transaction, then mark this as a special - ** "transaction savepoint". */ - if( db->autoCommit ){ - db->autoCommit = 0; - db->isTransactionSavepoint = 1; - }else{ - db->nSavepoint++; - } - - /* Link the new savepoint into the database handle's list. */ - u.aq.pNew->pNext = db->pSavepoint; - db->pSavepoint = u.aq.pNew; - u.aq.pNew->nDeferredCons = db->nDeferredCons; - } - } - }else{ - u.aq.iSavepoint = 0; - - /* Find the named savepoint. If there is no such savepoint, then an - ** an error is returned to the user. */ - for( - u.aq.pSavepoint = db->pSavepoint; - u.aq.pSavepoint && sqlite3StrICmp(u.aq.pSavepoint->zName, u.aq.zName); - u.aq.pSavepoint = u.aq.pSavepoint->pNext - ){ - u.aq.iSavepoint++; - } - if( !u.aq.pSavepoint ){ - sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", u.aq.zName); - rc = SQLITE_ERROR; - }else if( - db->writeVdbeCnt>0 || (u.aq.p1==SAVEPOINT_ROLLBACK && db->activeVdbeCnt>1) - ){ - /* It is not possible to release (commit) a savepoint if there are - ** active write statements. It is not possible to rollback a savepoint - ** if there are any active statements at all. - */ - sqlite3SetString(&p->zErrMsg, db, - "cannot %s savepoint - SQL statements in progress", - (u.aq.p1==SAVEPOINT_ROLLBACK ? "rollback": "release") - ); - rc = SQLITE_BUSY; - }else{ - - /* Determine whether or not this is a transaction savepoint. If so, - ** and this is a RELEASE command, then the current transaction - ** is committed. - */ - int isTransaction = u.aq.pSavepoint->pNext==0 && db->isTransactionSavepoint; - if( isTransaction && u.aq.p1==SAVEPOINT_RELEASE ){ - if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ - goto vdbe_return; - } - db->autoCommit = 1; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = pc; - db->autoCommit = 0; - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } - db->isTransactionSavepoint = 0; - rc = p->rc; - }else{ - u.aq.iSavepoint = db->nSavepoint - u.aq.iSavepoint - 1; - for(u.aq.ii=0; u.aq.iinDb; u.aq.ii++){ - rc = sqlite3BtreeSavepoint(db->aDb[u.aq.ii].pBt, u.aq.p1, u.aq.iSavepoint); - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - } - if( u.aq.p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ - sqlite3ExpirePreparedStatements(db); - sqlite3ResetInternalSchema(db, 0); - db->flags = (db->flags | SQLITE_InternChanges); - } - } - - /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all - ** savepoints nested inside of the savepoint being operated on. */ - while( db->pSavepoint!=u.aq.pSavepoint ){ - u.aq.pTmp = db->pSavepoint; - db->pSavepoint = u.aq.pTmp->pNext; - sqlite3DbFree(db, u.aq.pTmp); - db->nSavepoint--; - } - - /* If it is a RELEASE, then destroy the savepoint being operated on - ** too. If it is a ROLLBACK TO, then set the number of deferred - ** constraint violations present in the database to the value stored - ** when the savepoint was created. */ - if( u.aq.p1==SAVEPOINT_RELEASE ){ - assert( u.aq.pSavepoint==db->pSavepoint ); - db->pSavepoint = u.aq.pSavepoint->pNext; - sqlite3DbFree(db, u.aq.pSavepoint); - if( !isTransaction ){ - db->nSavepoint--; - } - }else{ - db->nDeferredCons = u.aq.pSavepoint->nDeferredCons; - } - } - } - - break; -} - -/* Opcode: AutoCommit P1 P2 * * * -** -** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll -** back any currently active btree transactions. If there are any active -** VMs (apart from this one), then a ROLLBACK fails. A COMMIT fails if -** there are active writing VMs or active VMs that use shared cache. -** -** This instruction causes the VM to halt. -*/ -case OP_AutoCommit: { -#if 0 /* local variables moved into u.ar */ - int desiredAutoCommit; - int iRollback; - int turnOnAC; -#endif /* local variables moved into u.ar */ - - u.ar.desiredAutoCommit = pOp->p1; - u.ar.iRollback = pOp->p2; - u.ar.turnOnAC = u.ar.desiredAutoCommit && !db->autoCommit; - assert( u.ar.desiredAutoCommit==1 || u.ar.desiredAutoCommit==0 ); - assert( u.ar.desiredAutoCommit==1 || u.ar.iRollback==0 ); - assert( db->activeVdbeCnt>0 ); /* At least this one VM is active */ - - if( u.ar.turnOnAC && u.ar.iRollback && db->activeVdbeCnt>1 ){ - /* If this instruction implements a ROLLBACK and other VMs are - ** still running, and a transaction is active, return an error indicating - ** that the other VMs must complete first. - */ - sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else if( u.ar.turnOnAC && !u.ar.iRollback && db->writeVdbeCnt>0 ){ - /* If this instruction implements a COMMIT and other VMs are writing - ** return an error indicating that the other VMs must complete first. - */ - sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else if( u.ar.desiredAutoCommit!=db->autoCommit ){ - if( u.ar.iRollback ){ - assert( u.ar.desiredAutoCommit==1 ); - sqlite3RollbackAll(db); - db->autoCommit = 1; - }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ - goto vdbe_return; - }else{ - db->autoCommit = (u8)u.ar.desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = pc; - db->autoCommit = (u8)(1-u.ar.desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } - } - assert( db->nStatement==0 ); - sqlite3CloseSavepoints(db); - if( p->rc==SQLITE_OK ){ - rc = SQLITE_DONE; - }else{ - rc = SQLITE_ERROR; - } - goto vdbe_return; - }else{ - sqlite3SetString(&p->zErrMsg, db, - (!u.ar.desiredAutoCommit)?"cannot start a transaction within a transaction":( - (u.ar.iRollback)?"cannot rollback - no transaction is active": - "cannot commit - no transaction is active")); - - rc = SQLITE_ERROR; - } - break; -} - -/* Opcode: Transaction P1 P2 * * * -** -** Begin a transaction. The transaction ends when a Commit or Rollback -** opcode is encountered. Depending on the ON CONFLICT setting, the -** transaction might also be rolled back if an error is encountered. -** -** P1 is the index of the database file on which the transaction is -** started. Index 0 is the main database file and index 1 is the -** file used for temporary tables. Indices of 2 or more are used for -** attached databases. -** -** If P2 is non-zero, then a write-transaction is started. A RESERVED lock is -** obtained on the database file when a write-transaction is started. No -** other process can start another write transaction while this transaction is -** underway. Starting a write transaction also creates a rollback journal. A -** write transaction must be started before any changes can be made to the -** database. If P2 is 2 or greater then an EXCLUSIVE lock is also obtained -** on the file. -** -** If a write-transaction is started and the Vdbe.usesStmtJournal flag is -** true (this flag is set if the Vdbe may modify more than one row and may -** throw an ABORT exception), a statement transaction may also be opened. -** More specifically, a statement transaction is opened iff the database -** connection is currently not in autocommit mode, or if there are other -** active statements. A statement transaction allows the affects of this -** VDBE to be rolled back after an error without having to roll back the -** entire transaction. If no error is encountered, the statement transaction -** will automatically commit when the VDBE halts. -** -** If P2 is zero, then a read-lock is obtained on the database file. -*/ -case OP_Transaction: { -#if 0 /* local variables moved into u.as */ - Btree *pBt; -#endif /* local variables moved into u.as */ - - assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (1<p1))!=0 ); - u.as.pBt = db->aDb[pOp->p1].pBt; - - if( u.as.pBt ){ - rc = sqlite3BtreeBeginTrans(u.as.pBt, pOp->p2); - if( rc==SQLITE_BUSY ){ - p->pc = pc; - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - - if( pOp->p2 && p->usesStmtJournal - && (db->autoCommit==0 || db->activeVdbeCnt>1) - ){ - assert( sqlite3BtreeIsInTrans(u.as.pBt) ); - if( p->iStatement==0 ){ - assert( db->nStatement>=0 && db->nSavepoint>=0 ); - db->nStatement++; - p->iStatement = db->nSavepoint + db->nStatement; - } - rc = sqlite3BtreeBeginStmt(u.as.pBt, p->iStatement); - - /* Store the current value of the database handles deferred constraint - ** counter. If the statement transaction needs to be rolled back, - ** the value of this counter needs to be restored too. */ - p->nStmtDefCons = db->nDeferredCons; - } - } - break; -} - -/* Opcode: ReadCookie P1 P2 P3 * * -** -** Read cookie number P3 from database P1 and write it into register P2. -** P3==1 is the schema version. P3==2 is the database format. -** P3==3 is the recommended pager cache size, and so forth. P1==0 is -** the main database file and P1==1 is the database file used to store -** temporary tables. -** -** There must be a read-lock on the database (either a transaction -** must be started or there must be an open cursor) before -** executing this instruction. -*/ -case OP_ReadCookie: { /* out2-prerelease */ -#if 0 /* local variables moved into u.at */ - int iMeta; - int iDb; - int iCookie; -#endif /* local variables moved into u.at */ - - u.at.iDb = pOp->p1; - u.at.iCookie = pOp->p3; - assert( pOp->p3=0 && u.at.iDbnDb ); - assert( db->aDb[u.at.iDb].pBt!=0 ); - assert( (p->btreeMask & (1<aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta); - pOut->u.i = u.at.iMeta; - break; -} - -/* Opcode: SetCookie P1 P2 P3 * * -** -** Write the content of register P3 (interpreted as an integer) -** into cookie number P2 of database P1. P2==1 is the schema version. -** P2==2 is the database format. P2==3 is the recommended pager cache -** size, and so forth. P1==0 is the main database file and P1==1 is the -** database file used to store temporary tables. -** -** A transaction must be started before executing this opcode. -*/ -case OP_SetCookie: { /* in3 */ -#if 0 /* local variables moved into u.au */ - Db *pDb; -#endif /* local variables moved into u.au */ - assert( pOp->p2p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (1<p1))!=0 ); - u.au.pDb = &db->aDb[pOp->p1]; - assert( u.au.pDb->pBt!=0 ); - pIn3 = &aMem[pOp->p3]; - sqlite3VdbeMemIntegerify(pIn3); - /* See note about index shifting on OP_ReadCookie */ - rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i); - if( pOp->p2==BTREE_SCHEMA_VERSION ){ - /* When the schema cookie changes, record the new cookie internally */ - u.au.pDb->pSchema->schema_cookie = (int)pIn3->u.i; - db->flags |= SQLITE_InternChanges; - }else if( pOp->p2==BTREE_FILE_FORMAT ){ - /* Record changes in the file format */ - u.au.pDb->pSchema->file_format = (u8)pIn3->u.i; - } - if( pOp->p1==1 ){ - /* Invalidate all prepared statements whenever the TEMP database - ** schema is changed. Ticket #1644 */ - sqlite3ExpirePreparedStatements(db); - p->expired = 0; - } - break; -} - -/* Opcode: VerifyCookie P1 P2 * -** -** Check the value of global database parameter number 0 (the -** schema version) and make sure it is equal to P2. -** P1 is the database number which is 0 for the main database file -** and 1 for the file holding temporary tables and some higher number -** for auxiliary databases. -** -** The cookie changes its value whenever the database schema changes. -** This operation is used to detect when that the cookie has changed -** and that the current process needs to reread the schema. -** -** Either a transaction needs to have been started or an OP_Open needs -** to be executed (to establish a read lock) before this opcode is -** invoked. -*/ -case OP_VerifyCookie: { -#if 0 /* local variables moved into u.av */ - int iMeta; - Btree *pBt; -#endif /* local variables moved into u.av */ - assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (1<p1))!=0 ); - u.av.pBt = db->aDb[pOp->p1].pBt; - if( u.av.pBt ){ - sqlite3BtreeGetMeta(u.av.pBt, BTREE_SCHEMA_VERSION, (u32 *)&u.av.iMeta); - }else{ - u.av.iMeta = 0; - } - if( u.av.iMeta!=pOp->p2 ){ - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); - /* If the schema-cookie from the database file matches the cookie - ** stored with the in-memory representation of the schema, do - ** not reload the schema from the database file. - ** - ** If virtual-tables are in use, this is not just an optimization. - ** Often, v-tables store their data in other SQLite tables, which - ** are queried from within xNext() and other v-table methods using - ** prepared queries. If such a query is out-of-date, we do not want to - ** discard the database schema, as the user code implementing the - ** v-table would have to be ready for the sqlite3_vtab structure itself - ** to be invalidated whenever sqlite3_step() is called from within - ** a v-table method. - */ - if( db->aDb[pOp->p1].pSchema->schema_cookie!=u.av.iMeta ){ - sqlite3ResetInternalSchema(db, pOp->p1); - } - - sqlite3ExpirePreparedStatements(db); - rc = SQLITE_SCHEMA; - } - break; -} - -/* Opcode: OpenRead P1 P2 P3 P4 P5 -** -** Open a read-only cursor for the database table whose root page is -** P2 in a database file. The database file is determined by P3. -** P3==0 means the main database, P3==1 means the database used for -** temporary tables, and P3>1 means used the corresponding attached -** database. Give the new cursor an identifier of P1. The P1 -** values need not be contiguous but all P1 values should be small integers. -** It is an error for P1 to be negative. +/* +** Execute as much of a VDBE program as we can then return. ** -** If P5!=0 then use the content of register P2 as the root page, not -** the value of P2 itself. +** sqlite3VdbeMakeReady() must be called before this routine in order to +** close the program with a final OP_Halt and to set up the callbacks +** and the error message pointer. ** -** There will be a read lock on the database whenever there is an -** open cursor. If the database was unlocked prior to this instruction -** then a read lock is acquired as part of this instruction. A read -** lock allows other processes to read the database but prohibits -** any other process from modifying the database. The read lock is -** released when all cursors are closed. If this instruction attempts -** to get a read lock but fails, the script terminates with an -** SQLITE_BUSY error code. +** Whenever a row or result data is available, this routine will either +** invoke the result callback (if there is one) or return with +** SQLITE_ROW. ** -** The P4 value may be either an integer (P4_INT32) or a pointer to -** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table. +** If an attempt is made to open a locked database, then this routine +** will either invoke the busy callback (if there is one) or it will +** return SQLITE_BUSY. ** -** See also OpenWrite. -*/ -/* Opcode: OpenWrite P1 P2 P3 P4 P5 +** If an error occurs, an error message is written to memory obtained +** from sqlite3_malloc() and p->zErrMsg is made to point to that memory. +** The error code is stored in p->rc and this routine returns SQLITE_ERROR. ** -** Open a read/write cursor named P1 on the table or index whose root -** page is P2. Or if P5!=0 use the content of register P2 to find the -** root page. +** If the callback ever returns non-zero, then the program exits +** immediately. There will be no error message but the p->rc field is +** set to SQLITE_ABORT and this routine will return SQLITE_ERROR. ** -** The P4 value may be either an integer (P4_INT32) or a pointer to -** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table, or to the -** largest index of any column of the table that is actually used. +** A memory allocation error causes p->rc to be set to SQLITE_NOMEM and this +** routine to return SQLITE_ERROR. ** -** This instruction works just like OpenRead except that it opens the cursor -** in read/write mode. For a given table, there can be one or more read-only -** cursors or a single read/write cursor but not both. +** Other fatal errors return SQLITE_ERROR. ** -** See also OpenRead. +** After this routine has finished, sqlite3VdbeFinalize() should be +** used to clean up the mess that was left behind. */ -case OP_OpenRead: -case OP_OpenWrite: { -#if 0 /* local variables moved into u.aw */ - int nField; - KeyInfo *pKeyInfo; - int p2; - int iDb; - int wrFlag; - Btree *pX; - VdbeCursor *pCur; - Db *pDb; -#endif /* local variables moved into u.aw */ - - if( p->expired ){ - rc = SQLITE_ABORT; - break; - } - - u.aw.nField = 0; - u.aw.pKeyInfo = 0; - u.aw.p2 = pOp->p2; - u.aw.iDb = pOp->p3; - assert( u.aw.iDb>=0 && u.aw.iDbnDb ); - assert( (p->btreeMask & (1<aDb[u.aw.iDb]; - u.aw.pX = u.aw.pDb->pBt; - assert( u.aw.pX!=0 ); - if( pOp->opcode==OP_OpenWrite ){ - u.aw.wrFlag = 1; - if( u.aw.pDb->pSchema->file_format < p->minWriteFileFormat ){ - p->minWriteFileFormat = u.aw.pDb->pSchema->file_format; - } - }else{ - u.aw.wrFlag = 0; - } - if( pOp->p5 ){ - assert( u.aw.p2>0 ); - assert( u.aw.p2<=p->nMem ); - pIn2 = &aMem[u.aw.p2]; - assert( memIsValid(pIn2) ); - assert( (pIn2->flags & MEM_Int)!=0 ); - sqlite3VdbeMemIntegerify(pIn2); - u.aw.p2 = (int)pIn2->u.i; - /* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and - ** that opcode will always set the u.aw.p2 value to 2 or more or else fail. - ** If there were a failure, the prepared statement would have halted - ** before reaching this instruction. */ - if( NEVER(u.aw.p2<2) ) { - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; - } - } - if( pOp->p4type==P4_KEYINFO ){ - u.aw.pKeyInfo = pOp->p4.pKeyInfo; - u.aw.pKeyInfo->enc = ENC(p->db); - u.aw.nField = u.aw.pKeyInfo->nField+1; - }else if( pOp->p4type==P4_INT32 ){ - u.aw.nField = pOp->p4.i; - } - assert( pOp->p1>=0 ); - u.aw.pCur = allocateCursor(p, pOp->p1, u.aw.nField, u.aw.iDb, 1); - if( u.aw.pCur==0 ) goto no_mem; - u.aw.pCur->nullRow = 1; - u.aw.pCur->isOrdered = 1; - rc = sqlite3BtreeCursor(u.aw.pX, u.aw.p2, u.aw.wrFlag, u.aw.pKeyInfo, u.aw.pCur->pCursor); - u.aw.pCur->pKeyInfo = u.aw.pKeyInfo; - - /* Since it performs no memory allocation or IO, the only values that - ** sqlite3BtreeCursor() may return are SQLITE_EMPTY and SQLITE_OK. - ** SQLITE_EMPTY is only returned when attempting to open the table - ** rooted at page 1 of a zero-byte database. */ - assert( rc==SQLITE_EMPTY || rc==SQLITE_OK ); - if( rc==SQLITE_EMPTY ){ - u.aw.pCur->pCursor = 0; - rc = SQLITE_OK; - } - - /* Set the VdbeCursor.isTable and isIndex variables. Previous versions of - ** SQLite used to check if the root-page flags were sane at this point - ** and report database corruption if they were not, but this check has - ** since moved into the btree layer. */ - u.aw.pCur->isTable = pOp->p4type!=P4_KEYINFO; - u.aw.pCur->isIndex = !u.aw.pCur->isTable; - break; -} - -/* Opcode: OpenEphemeral P1 P2 * P4 * -** -** Open a new cursor P1 to a transient table. -** The cursor is always opened read/write even if -** the main database is read-only. The ephemeral -** table is deleted automatically when the cursor is closed. -** -** P2 is the number of columns in the ephemeral table. -** The cursor points to a BTree table if P4==0 and to a BTree index -** if P4 is not 0. If P4 is not NULL, it points to a KeyInfo structure -** that defines the format of keys in the index. -** -** This opcode was once called OpenTemp. But that created -** confusion because the term "temp table", might refer either -** to a TEMP table at the SQL level, or to a table opened by -** this opcode. Then this opcode was call OpenVirtual. But -** that created confusion with the whole virtual-table idea. -*/ -/* Opcode: OpenAutoindex P1 P2 * P4 * -** -** This opcode works the same as OP_OpenEphemeral. It has a -** different name to distinguish its use. Tables created using -** by this opcode will be used for automatically created transient -** indices in joins. -*/ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { -#if 0 /* local variables moved into u.ax */ - VdbeCursor *pCx; -#endif /* local variables moved into u.ax */ - static const int vfsFlags = - SQLITE_OPEN_READWRITE | - SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_TRANSIENT_DB; +SQLITE_PRIVATE int sqlite3VdbeExec( + Vdbe *p /* The VDBE */ +){ + int pc=0; /* The program counter */ + Op *aOp = p->aOp; /* Copy of p->aOp */ + Op *pOp; /* Current operation */ + int rc = SQLITE_OK; /* Value to return */ + sqlite3 *db = p->db; /* The database */ + u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ + u8 encoding = ENC(db); /* The database encoding */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + int checkProgress; /* True if progress callbacks are enabled */ + int nProgressOps = 0; /* Opcodes executed since progress callback. */ +#endif + Mem *aMem = p->aMem; /* Copy of p->aMem */ + Mem *pIn1 = 0; /* 1st input operand */ + Mem *pIn2 = 0; /* 2nd input operand */ + Mem *pIn3 = 0; /* 3rd input operand */ + Mem *pOut = 0; /* Output operand */ + int iCompare = 0; /* Result of last OP_Compare operation */ + int *aPermute = 0; /* Permutation of columns for OP_Compare */ + i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ +#ifdef VDBE_PROFILE + u64 start; /* CPU clock count at start of opcode */ + int origPc; /* Program counter at start of opcode */ +#endif + /******************************************************************** + ** Automatically generated code + ** + ** The following union is automatically generated by the + ** vdbe-compress.tcl script. The purpose of this union is to + ** reduce the amount of stack space required by this function. + ** See comments in the vdbe-compress.tcl script for details. + */ + union vdbeExecUnion { + struct OP_Yield_stack_vars { + int pcDest; + } aa; + struct OP_Variable_stack_vars { + Mem *pVar; /* Value being transferred */ + } ab; + struct OP_Move_stack_vars { + char *zMalloc; /* Holding variable for allocated memory */ + int n; /* Number of registers left to copy */ + int p1; /* Register to copy from */ + int p2; /* Register to copy to */ + } ac; + struct OP_ResultRow_stack_vars { + Mem *pMem; + int i; + } ad; + struct OP_Concat_stack_vars { + i64 nByte; + } ae; + struct OP_Remainder_stack_vars { + int flags; /* Combined MEM_* flags from both inputs */ + i64 iA; /* Integer value of left operand */ + i64 iB; /* Integer value of right operand */ + double rA; /* Real value of left operand */ + double rB; /* Real value of right operand */ + } af; + struct OP_Function_stack_vars { + int i; + Mem *pArg; + sqlite3_context ctx; + sqlite3_value **apVal; + int n; + } ag; + struct OP_ShiftRight_stack_vars { + i64 iA; + u64 uA; + i64 iB; + u8 op; + } ah; + struct OP_Ge_stack_vars { + int res; /* Result of the comparison of pIn1 against pIn3 */ + char affinity; /* Affinity to use for comparison */ + u16 flags1; /* Copy of initial value of pIn1->flags */ + u16 flags3; /* Copy of initial value of pIn3->flags */ + } ai; + struct OP_Compare_stack_vars { + int n; + int i; + int p1; + int p2; + const KeyInfo *pKeyInfo; + int idx; + CollSeq *pColl; /* Collating sequence to use on this term */ + int bRev; /* True for DESCENDING sort order */ + } aj; + struct OP_Or_stack_vars { + int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + } ak; + struct OP_IfNot_stack_vars { + int c; + } al; + struct OP_Column_stack_vars { + u32 payloadSize; /* Number of bytes in the record */ + i64 payloadSize64; /* Number of bytes in the record */ + int p1; /* P1 value of the opcode */ + int p2; /* column number to retrieve */ + VdbeCursor *pC; /* The VDBE cursor */ + char *zRec; /* Pointer to complete record-data */ + BtCursor *pCrsr; /* The BTree cursor */ + u32 *aType; /* aType[i] holds the numeric type of the i-th column */ + u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ + int nField; /* number of fields in the record */ + int len; /* The length of the serialized data for the column */ + int i; /* Loop counter */ + char *zData; /* Part of the record being decoded */ + Mem *pDest; /* Where to write the extracted value */ + Mem sMem; /* For storing the record being decoded */ + u8 *zIdx; /* Index into header */ + u8 *zEndHdr; /* Pointer to first byte after the header */ + u32 offset; /* Offset into the data */ + u32 szField; /* Number of bytes in the content of a field */ + int szHdr; /* Size of the header size field at start of record */ + int avail; /* Number of bytes of available data */ + Mem *pReg; /* PseudoTable input register */ + } am; + struct OP_Affinity_stack_vars { + const char *zAffinity; /* The affinity to be applied */ + char cAff; /* A single character of affinity */ + } an; + struct OP_MakeRecord_stack_vars { + u8 *zNewRecord; /* A buffer to hold the data for the new record */ + Mem *pRec; /* The new record */ + u64 nData; /* Number of bytes of data space */ + int nHdr; /* Number of bytes of header space */ + i64 nByte; /* Data space required for this record */ + int nZero; /* Number of zero bytes at the end of the record */ + int nVarint; /* Number of bytes in a varint */ + u32 serial_type; /* Type field */ + Mem *pData0; /* First field to be combined into the record */ + Mem *pLast; /* Last field of the record */ + int nField; /* Number of fields in the record */ + char *zAffinity; /* The affinity string for the record */ + int file_format; /* File format to use for encoding */ + int i; /* Space used in zNewRecord[] */ + int len; /* Length of a field */ + } ao; + struct OP_Count_stack_vars { + i64 nEntry; + BtCursor *pCrsr; + } ap; + struct OP_Savepoint_stack_vars { + int p1; /* Value of P1 operand */ + char *zName; /* Name of savepoint */ + int nName; + Savepoint *pNew; + Savepoint *pSavepoint; + Savepoint *pTmp; + int iSavepoint; + int ii; + } aq; + struct OP_AutoCommit_stack_vars { + int desiredAutoCommit; + int iRollback; + int turnOnAC; + } ar; + struct OP_Transaction_stack_vars { + Btree *pBt; + } as; + struct OP_ReadCookie_stack_vars { + int iMeta; + int iDb; + int iCookie; + } at; + struct OP_SetCookie_stack_vars { + Db *pDb; + } au; + struct OP_VerifyCookie_stack_vars { + int iMeta; + int iGen; + Btree *pBt; + } av; + struct OP_OpenWrite_stack_vars { + int nField; + KeyInfo *pKeyInfo; + int p2; + int iDb; + int wrFlag; + Btree *pX; + VdbeCursor *pCur; + Db *pDb; + } aw; + struct OP_OpenEphemeral_stack_vars { + VdbeCursor *pCx; + } ax; + struct OP_OpenPseudo_stack_vars { + VdbeCursor *pCx; + } ay; + struct OP_SeekGt_stack_vars { + int res; + int oc; + VdbeCursor *pC; + UnpackedRecord r; + int nField; + i64 iKey; /* The rowid we are to seek to */ + } az; + struct OP_Seek_stack_vars { + VdbeCursor *pC; + } ba; + struct OP_Found_stack_vars { + int alreadyExists; + VdbeCursor *pC; + int res; + UnpackedRecord *pIdxKey; + UnpackedRecord r; + char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; + } bb; + struct OP_IsUnique_stack_vars { + u16 ii; + VdbeCursor *pCx; + BtCursor *pCrsr; + u16 nField; + Mem *aMx; + UnpackedRecord r; /* B-Tree index search key */ + i64 R; /* Rowid stored in register P3 */ + } bc; + struct OP_NotExists_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + u64 iKey; + } bd; + struct OP_NewRowid_stack_vars { + i64 v; /* The new rowid */ + VdbeCursor *pC; /* Cursor of table to get the new rowid */ + int res; /* Result of an sqlite3BtreeLast() */ + int cnt; /* Counter to limit the number of searches */ + Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ + VdbeFrame *pFrame; /* Root frame of VDBE */ + } be; + struct OP_InsertInt_stack_vars { + Mem *pData; /* MEM cell holding data for the record to be inserted */ + Mem *pKey; /* MEM cell holding key for the record */ + i64 iKey; /* The integer ROWID or key for the record to be inserted */ + VdbeCursor *pC; /* Cursor to table into which insert is written */ + int nZero; /* Number of zero-bytes to append */ + int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ + const char *zDb; /* database name - used by the update hook */ + const char *zTbl; /* Table name - used by the opdate hook */ + int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ + } bf; + struct OP_Delete_stack_vars { + i64 iKey; + VdbeCursor *pC; + } bg; + struct OP_RowData_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + u32 n; + i64 n64; + } bh; + struct OP_Rowid_stack_vars { + VdbeCursor *pC; + i64 v; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + } bi; + struct OP_NullRow_stack_vars { + VdbeCursor *pC; + } bj; + struct OP_Last_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + } bk; + struct OP_Rewind_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + } bl; + struct OP_Next_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + } bm; + struct OP_IdxInsert_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + int nKey; + const char *zKey; + } bn; + struct OP_IdxDelete_stack_vars { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + UnpackedRecord r; + } bo; + struct OP_IdxRowid_stack_vars { + BtCursor *pCrsr; + VdbeCursor *pC; + i64 rowid; + } bp; + struct OP_IdxGE_stack_vars { + VdbeCursor *pC; + int res; + UnpackedRecord r; + } bq; + struct OP_Destroy_stack_vars { + int iMoved; + int iCnt; + Vdbe *pVdbe; + int iDb; + } br; + struct OP_Clear_stack_vars { + int nChange; + } bs; + struct OP_CreateTable_stack_vars { + int pgno; + int flags; + Db *pDb; + } bt; + struct OP_ParseSchema_stack_vars { + int iDb; + const char *zMaster; + char *zSql; + InitData initData; + } bu; + struct OP_IntegrityCk_stack_vars { + int nRoot; /* Number of tables to check. (Number of root pages.) */ + int *aRoot; /* Array of rootpage numbers for tables to be checked */ + int j; /* Loop counter */ + int nErr; /* Number of errors reported */ + char *z; /* Text of the error report */ + Mem *pnErr; /* Register keeping track of errors remaining */ + } bv; + struct OP_RowSetRead_stack_vars { + i64 val; + } bw; + struct OP_RowSetTest_stack_vars { + int iSet; + int exists; + } bx; + struct OP_Program_stack_vars { + int nMem; /* Number of memory registers for sub-program */ + int nByte; /* Bytes of runtime space required for sub-program */ + Mem *pRt; /* Register to allocate runtime space */ + Mem *pMem; /* Used to iterate through memory cells */ + Mem *pEnd; /* Last memory cell in new array */ + VdbeFrame *pFrame; /* New vdbe frame to execute in */ + SubProgram *pProgram; /* Sub-program to execute */ + void *t; /* Token identifying trigger */ + } by; + struct OP_Param_stack_vars { + VdbeFrame *pFrame; + Mem *pIn; + } bz; + struct OP_MemMax_stack_vars { + Mem *pIn1; + VdbeFrame *pFrame; + } ca; + struct OP_AggStep_stack_vars { + int n; + int i; + Mem *pMem; + Mem *pRec; + sqlite3_context ctx; + sqlite3_value **apVal; + } cb; + struct OP_AggFinal_stack_vars { + Mem *pMem; + } cc; + struct OP_Checkpoint_stack_vars { + int i; /* Loop counter */ + int aRes[3]; /* Results */ + Mem *pMem; /* Write results here */ + } cd; + struct OP_JournalMode_stack_vars { + Btree *pBt; /* Btree to change journal mode of */ + Pager *pPager; /* Pager associated with pBt */ + int eNew; /* New journal mode */ + int eOld; /* The old journal mode */ + const char *zFilename; /* Name of database file for pPager */ + } ce; + struct OP_IncrVacuum_stack_vars { + Btree *pBt; + } cf; + struct OP_VBegin_stack_vars { + VTable *pVTab; + } cg; + struct OP_VOpen_stack_vars { + VdbeCursor *pCur; + sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab *pVtab; + sqlite3_module *pModule; + } ch; + struct OP_VFilter_stack_vars { + int nArg; + int iQuery; + const sqlite3_module *pModule; + Mem *pQuery; + Mem *pArgc; + sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab *pVtab; + VdbeCursor *pCur; + int res; + int i; + Mem **apArg; + } ci; + struct OP_VColumn_stack_vars { + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + Mem *pDest; + sqlite3_context sContext; + } cj; + struct OP_VNext_stack_vars { + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + int res; + VdbeCursor *pCur; + } ck; + struct OP_VRename_stack_vars { + sqlite3_vtab *pVtab; + Mem *pName; + } cl; + struct OP_VUpdate_stack_vars { + sqlite3_vtab *pVtab; + sqlite3_module *pModule; + int nArg; + int i; + sqlite_int64 rowid; + Mem **apArg; + Mem *pX; + } cm; + struct OP_Trace_stack_vars { + char *zTrace; + char *z; + } cn; + } u; + /* End automatically generated code + ********************************************************************/ - assert( pOp->p1>=0 ); - u.ax.pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); - if( u.ax.pCx==0 ) goto no_mem; - u.ax.pCx->nullRow = 1; - rc = sqlite3BtreeOpen(0, db, &u.ax.pCx->pBt, - BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(u.ax.pCx->pBt, 1); + assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ + sqlite3VdbeEnter(p); + if( p->rc==SQLITE_NOMEM ){ + /* This happens if a malloc() inside a call to sqlite3_column_text() or + ** sqlite3_column_text16() failed. */ + goto no_mem; } - if( rc==SQLITE_OK ){ - /* If a transient index is required, create it by calling - ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before - ** opening it. If a transient table is required, just use the - ** automatically created table with root-page 1 (an BLOB_INTKEY table). - */ - if( pOp->p4.pKeyInfo ){ - int pgno; - assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY); - if( rc==SQLITE_OK ){ - assert( pgno==MASTER_ROOT+1 ); - rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1, - (KeyInfo*)pOp->p4.z, u.ax.pCx->pCursor); - u.ax.pCx->pKeyInfo = pOp->p4.pKeyInfo; - u.ax.pCx->pKeyInfo->enc = ENC(p->db); - } - u.ax.pCx->isTable = 0; - }else{ - rc = sqlite3BtreeCursor(u.ax.pCx->pBt, MASTER_ROOT, 1, 0, u.ax.pCx->pCursor); - u.ax.pCx->isTable = 1; + assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + p->rc = SQLITE_OK; + assert( p->explain==0 ); + p->pResultSet = 0; + db->busyHandler.nBusy = 0; + CHECK_FOR_INTERRUPT; + sqlite3VdbeIOTraceSql(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + checkProgress = db->xProgress!=0; +#endif +#ifdef SQLITE_DEBUG + sqlite3BeginBenignMalloc(); + if( p->pc==0 && (p->db->flags & SQLITE_VdbeListing)!=0 ){ + int i; + printf("VDBE Program Listing:\n"); + sqlite3VdbePrintSql(p); + for(i=0; inOp; i++){ + sqlite3VdbePrintOp(stdout, i, &aOp[i]); } } - u.ax.pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); - u.ax.pCx->isIndex = !u.ax.pCx->isTable; - break; -} + sqlite3EndBenignMalloc(); +#endif + for(pc=p->pc; rc==SQLITE_OK; pc++){ + assert( pc>=0 && pcnOp ); + if( db->mallocFailed ) goto no_mem; +#ifdef VDBE_PROFILE + origPc = pc; + start = sqlite3Hwtime(); +#endif + pOp = &aOp[pc]; -/* Opcode: OpenPseudo P1 P2 P3 * * + /* Only allow tracing if SQLITE_DEBUG is defined. + */ +#ifdef SQLITE_DEBUG + if( p->trace ){ + if( pc==0 ){ + printf("VDBE Execution Trace:\n"); + sqlite3VdbePrintSql(p); + } + sqlite3VdbePrintOp(p->trace, pc, pOp); + } +#endif + + + /* Check to see if we need to simulate an interrupt. This only happens + ** if we have a special test build. + */ +#ifdef SQLITE_TEST + if( sqlite3_interrupt_count>0 ){ + sqlite3_interrupt_count--; + if( sqlite3_interrupt_count==0 ){ + sqlite3_interrupt(db); + } + } +#endif + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + /* Call the progress callback if it is configured and the required number + ** of VDBE ops have been executed (either since this invocation of + ** sqlite3VdbeExec() or since last time the progress callback was called). + ** If the progress callback returns non-zero, exit the virtual machine with + ** a return code SQLITE_ABORT. + */ + if( checkProgress ){ + if( db->nProgressOps==nProgressOps ){ + int prc; + prc = db->xProgress(db->pProgressArg); + if( prc!=0 ){ + rc = SQLITE_INTERRUPT; + goto vdbe_error_halt; + } + nProgressOps = 0; + } + nProgressOps++; + } +#endif + + /* On any opcode with the "out2-prerelase" tag, free any + ** external allocations out of mem[p2] and set mem[p2] to be + ** an undefined integer. Opcodes will either fill in the integer + ** value or convert mem[p2] to a different type. + */ + assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); + if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=p->nMem ); + pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); + sqlite3VdbeMemReleaseExternal(pOut); + pOut->flags = MEM_Int; + } + + /* Sanity checking on other operands */ +#ifdef SQLITE_DEBUG + if( (pOp->opflags & OPFLG_IN1)!=0 ){ + assert( pOp->p1>0 ); + assert( pOp->p1<=p->nMem ); + assert( memIsValid(&aMem[pOp->p1]) ); + REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); + } + if( (pOp->opflags & OPFLG_IN2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=p->nMem ); + assert( memIsValid(&aMem[pOp->p2]) ); + REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_IN3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=p->nMem ); + assert( memIsValid(&aMem[pOp->p3]) ); + REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); + } + if( (pOp->opflags & OPFLG_OUT2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_OUT3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p3]); + } +#endif + + switch( pOp->opcode ){ + +/***************************************************************************** +** What follows is a massive switch statement where each case implements a +** separate instruction in the virtual machine. If we follow the usual +** indentation conventions, each case should be indented by 6 spaces. But +** that is a lot of wasted space on the left margin. So the code within +** the switch statement will break with convention and be flush-left. Another +** big comment (similar to this one) will mark the point in the code where +** we transition back to normal indentation. ** -** Open a new cursor that points to a fake table that contains a single -** row of data. The content of that one row in the content of memory -** register P2. In other words, cursor P1 becomes an alias for the -** MEM_Blob content contained in register P2. +** The formatting of each case is important. The makefile for SQLite +** generates two C files "opcodes.h" and "opcodes.c" by scanning this +** file looking for lines that begin with "case OP_". The opcodes.h files +** will be filled with #defines that give unique integer values to each +** opcode and the opcodes.c file is filled with an array of strings where +** each string is the symbolic name for the corresponding opcode. If the +** case statement is followed by a comment of the form "/# same as ... #/" +** that comment is used to determine the particular value of the opcode. ** -** A pseudo-table created by this opcode is used to hold a single -** row output from the sorter so that the row can be decomposed into -** individual columns using the OP_Column opcode. The OP_Column opcode -** is the only cursor opcode that works with a pseudo-table. +** Other keywords in the comment that follows each case are used to +** construct the OPFLG_INITIALIZER value that initializes opcodeProperty[]. +** Keywords include: in1, in2, in3, out2_prerelease, out2, out3. See +** the mkopcodeh.awk script for additional information. ** -** P3 is the number of fields in the records that will be stored by -** the pseudo-table. -*/ -case OP_OpenPseudo: { -#if 0 /* local variables moved into u.ay */ - VdbeCursor *pCx; -#endif /* local variables moved into u.ay */ +** Documentation about VDBE opcodes is generated by scanning this file +** for lines of that contain "Opcode:". That line and all subsequent +** comment lines are used in the generation of the opcode.html documentation +** file. +** +** SUMMARY: +** +** Formatting is important to scripts that scan this file. +** Do not deviate from the formatting style currently in use. +** +*****************************************************************************/ - assert( pOp->p1>=0 ); - u.ay.pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); - if( u.ay.pCx==0 ) goto no_mem; - u.ay.pCx->nullRow = 1; - u.ay.pCx->pseudoTableReg = pOp->p2; - u.ay.pCx->isTable = 1; - u.ay.pCx->isIndex = 0; +/* Opcode: Goto * P2 * * * +** +** An unconditional jump to address P2. +** The next instruction executed will be +** the one at index P2 from the beginning of +** the program. +*/ +case OP_Goto: { /* jump */ + CHECK_FOR_INTERRUPT; + pc = pOp->p2 - 1; break; } -/* Opcode: Close P1 * * * * +/* Opcode: Gosub P1 P2 * * * ** -** Close a cursor previously opened as P1. If P1 is not -** currently open, this instruction is a no-op. +** Write the current address onto register P1 +** and then jump to address P2. */ -case OP_Close: { - assert( pOp->p1>=0 && pOp->p1nCursor ); - sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); - p->apCsr[pOp->p1] = 0; +case OP_Gosub: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Dyn)==0 ); + memAboutToChange(p, pIn1); + pIn1->flags = MEM_Int; + pIn1->u.i = pc; + REGISTER_TRACE(pOp->p1, pIn1); + pc = pOp->p2 - 1; break; } -/* Opcode: SeekGe P1 P2 P3 P4 * -** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as the key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. -** -** Reposition cursor P1 so that it points to the smallest entry that -** is greater than or equal to the key value. If there are no records -** greater than or equal to the key and P2 is not zero, then jump to P2. +/* Opcode: Return P1 * * * * ** -** See also: Found, NotFound, Distinct, SeekLt, SeekGt, SeekLe +** Jump to the next instruction after the address in register P1. */ -/* Opcode: SeekGt P1 P2 P3 P4 * -** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as a key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. -** -** Reposition cursor P1 so that it points to the smallest entry that -** is greater than the key value. If there are no records greater than -** the key and P2 is not zero, then jump to P2. +case OP_Return: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags & MEM_Int ); + pc = (int)pIn1->u.i; + break; +} + +/* Opcode: Yield P1 * * * * ** -** See also: Found, NotFound, Distinct, SeekLt, SeekGe, SeekLe +** Swap the program counter with the value in register P1. */ -/* Opcode: SeekLt P1 P2 P3 P4 * -** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as a key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. -** -** Reposition cursor P1 so that it points to the largest entry that -** is less than the key value. If there are no records less than -** the key and P2 is not zero, then jump to P2. +case OP_Yield: { /* in1 */ +#if 0 /* local variables moved into u.aa */ + int pcDest; +#endif /* local variables moved into u.aa */ + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Dyn)==0 ); + pIn1->flags = MEM_Int; + u.aa.pcDest = (int)pIn1->u.i; + pIn1->u.i = pc; + REGISTER_TRACE(pOp->p1, pIn1); + pc = u.aa.pcDest; + break; +} + +/* Opcode: HaltIfNull P1 P2 P3 P4 * ** -** See also: Found, NotFound, Distinct, SeekGt, SeekGe, SeekLe +** Check the value in register P3. If it is NULL then Halt using +** parameter P1, P2, and P4 as if this were a Halt instruction. If the +** value in register P3 is not NULL, then this routine is a no-op. */ -/* Opcode: SeekLe P1 P2 P3 P4 * +case OP_HaltIfNull: { /* in3 */ + pIn3 = &aMem[pOp->p3]; + if( (pIn3->flags & MEM_Null)==0 ) break; + /* Fall through into OP_Halt */ +} + +/* Opcode: Halt P1 P2 * P4 * ** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as a key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. +** Exit immediately. All open cursors, etc are closed +** automatically. ** -** Reposition cursor P1 so that it points to the largest entry that -** is less than or equal to the key value. If there are no records -** less than or equal to the key and P2 is not zero, then jump to P2. +** P1 is the result code returned by sqlite3_exec(), sqlite3_reset(), +** or sqlite3_finalize(). For a normal halt, this should be SQLITE_OK (0). +** For errors, it can be some other value. If P1!=0 then P2 will determine +** whether or not to rollback the current transaction. Do not rollback +** if P2==OE_Fail. Do the rollback if P2==OE_Rollback. If P2==OE_Abort, +** then back out all changes that have occurred during this execution of the +** VDBE, but do not rollback the transaction. ** -** See also: Found, NotFound, Distinct, SeekGt, SeekGe, SeekLt +** If P4 is not null then it is an error message string. +** +** There is an implied "Halt 0 0 0" instruction inserted at the very end of +** every program. So a jump past the last instruction of the program +** is the same as executing Halt. */ -case OP_SeekLt: /* jump, in3 */ -case OP_SeekLe: /* jump, in3 */ -case OP_SeekGe: /* jump, in3 */ -case OP_SeekGt: { /* jump, in3 */ -#if 0 /* local variables moved into u.az */ - int res; - int oc; - VdbeCursor *pC; - UnpackedRecord r; - int nField; - i64 iKey; /* The rowid we are to seek to */ -#endif /* local variables moved into u.az */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p2!=0 ); - u.az.pC = p->apCsr[pOp->p1]; - assert( u.az.pC!=0 ); - assert( u.az.pC->pseudoTableReg==0 ); - assert( OP_SeekLe == OP_SeekLt+1 ); - assert( OP_SeekGe == OP_SeekLt+2 ); - assert( OP_SeekGt == OP_SeekLt+3 ); - assert( u.az.pC->isOrdered ); - if( u.az.pC->pCursor!=0 ){ - u.az.oc = pOp->opcode; - u.az.pC->nullRow = 0; - if( u.az.pC->isTable ){ - /* The input value in P3 might be of any type: integer, real, string, - ** blob, or NULL. But it needs to be an integer before we can do - ** the seek, so covert it. */ - pIn3 = &aMem[pOp->p3]; - applyNumericAffinity(pIn3); - u.az.iKey = sqlite3VdbeIntValue(pIn3); - u.az.pC->rowidIsValid = 0; - - /* If the P3 value could not be converted into an integer without - ** loss of information, then special processing is required... */ - if( (pIn3->flags & MEM_Int)==0 ){ - if( (pIn3->flags & MEM_Real)==0 ){ - /* If the P3 value cannot be converted into any kind of a number, - ** then the seek is not possible, so jump to P2 */ - pc = pOp->p2 - 1; - break; - } - /* If we reach this point, then the P3 value must be a floating - ** point number. */ - assert( (pIn3->flags & MEM_Real)!=0 ); - - if( u.az.iKey==SMALLEST_INT64 && (pIn3->r<(double)u.az.iKey || pIn3->r>0) ){ - /* The P3 value is too large in magnitude to be expressed as an - ** integer. */ - u.az.res = 1; - if( pIn3->r<0 ){ - if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt ); - rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - } - }else{ - if( u.az.oc<=OP_SeekLe ){ assert( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe ); - rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - } - } - if( u.az.res ){ - pc = pOp->p2 - 1; - } - break; - }else if( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekGe ){ - /* Use the ceiling() function to convert real->int */ - if( pIn3->r > (double)u.az.iKey ) u.az.iKey++; - }else{ - /* Use the floor() function to convert real->int */ - assert( u.az.oc==OP_SeekLe || u.az.oc==OP_SeekGt ); - if( pIn3->r < (double)u.az.iKey ) u.az.iKey--; - } - } - rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, 0, (u64)u.az.iKey, 0, &u.az.res); - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - if( u.az.res==0 ){ - u.az.pC->rowidIsValid = 1; - u.az.pC->lastRowid = u.az.iKey; - } - }else{ - u.az.nField = pOp->p4.i; - assert( pOp->p4type==P4_INT32 ); - assert( u.az.nField>0 ); - u.az.r.pKeyInfo = u.az.pC->pKeyInfo; - u.az.r.nField = (u16)u.az.nField; - - /* The next line of code computes as follows, only faster: - ** if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekLe ){ - ** u.az.r.flags = UNPACKED_INCRKEY; - ** }else{ - ** u.az.r.flags = 0; - ** } - */ - u.az.r.flags = (u16)(UNPACKED_INCRKEY * (1 & (u.az.oc - OP_SeekLt))); - assert( u.az.oc!=OP_SeekGt || u.az.r.flags==UNPACKED_INCRKEY ); - assert( u.az.oc!=OP_SeekLe || u.az.r.flags==UNPACKED_INCRKEY ); - assert( u.az.oc!=OP_SeekGe || u.az.r.flags==0 ); - assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 ); - - u.az.r.aMem = &aMem[pOp->p3]; -#ifdef SQLITE_DEBUG - { int i; for(i=0; ipCursor, &u.az.r, 0, 0, &u.az.res); - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - u.az.pC->rowidIsValid = 0; - } - u.az.pC->deferredMoveto = 0; - u.az.pC->cacheStatus = CACHE_STALE; -#ifdef SQLITE_TEST - sqlite3_search_count++; -#endif - if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt ); - if( u.az.res<0 || (u.az.res==0 && u.az.oc==OP_SeekGt) ){ - rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - u.az.pC->rowidIsValid = 0; - }else{ - u.az.res = 0; - } - }else{ - assert( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe ); - if( u.az.res>0 || (u.az.res==0 && u.az.oc==OP_SeekLt) ){ - rc = sqlite3BtreePrevious(u.az.pC->pCursor, &u.az.res); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - u.az.pC->rowidIsValid = 0; - }else{ - /* u.az.res might be negative because the table is empty. Check to - ** see if this is the case. - */ - u.az.res = sqlite3BtreeEof(u.az.pC->pCursor); - } - } - assert( pOp->p2>0 ); - if( u.az.res ){ - pc = pOp->p2 - 1; +case OP_Halt: { + if( pOp->p1==SQLITE_OK && p->pFrame ){ + /* Halt the sub-program. Return control to the parent frame. */ + VdbeFrame *pFrame = p->pFrame; + p->pFrame = pFrame->pParent; + p->nFrame--; + sqlite3VdbeSetChanges(db, p->nChange); + pc = sqlite3VdbeFrameRestore(pFrame); + lastRowid = db->lastRowid; + if( pOp->p2==OE_Ignore ){ + /* Instruction pc is the OP_Program that invoked the sub-program + ** currently being halted. If the p2 instruction of this OP_Halt + ** instruction is set to OE_Ignore, then the sub-program is throwing + ** an IGNORE exception. In this case jump to the address specified + ** as the p2 of the calling OP_Program. */ + pc = p->aOp[pc].p2-1; } + aOp = p->aOp; + aMem = p->aMem; + break; + } + + p->rc = pOp->p1; + p->errorAction = (u8)pOp->p2; + p->pc = pc; + if( pOp->p4.z ){ + assert( p->rc!=SQLITE_OK ); + sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pc, p->zSql, pOp->p4.z); + }else if( p->rc ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(pOp->p1, "constraint failed at %d in [%s]", pc, p->zSql); + } + rc = sqlite3VdbeHalt(p); + assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); + if( rc==SQLITE_BUSY ){ + p->rc = rc = SQLITE_BUSY; }else{ - /* This happens when attempting to open the sqlite3_master table - ** for read access returns SQLITE_EMPTY. In this case always - ** take the jump (since there are no records in the table). - */ - pc = pOp->p2 - 1; + assert( rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ); + assert( rc==SQLITE_OK || db->nDeferredCons>0 ); + rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; } - break; + goto vdbe_return; } -/* Opcode: Seek P1 P2 * * * -** -** P1 is an open table cursor and P2 is a rowid integer. Arrange -** for P1 to move so that it points to the rowid given by P2. +/* Opcode: Integer P1 P2 * * * ** -** This is actually a deferred seek. Nothing actually happens until -** the cursor is used to read a record. That way, if no reads -** occur, no unnecessary I/O happens. +** The 32-bit integer value P1 is written into register P2. */ -case OP_Seek: { /* in2 */ -#if 0 /* local variables moved into u.ba */ - VdbeCursor *pC; -#endif /* local variables moved into u.ba */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.ba.pC = p->apCsr[pOp->p1]; - assert( u.ba.pC!=0 ); - if( ALWAYS(u.ba.pC->pCursor!=0) ){ - assert( u.ba.pC->isTable ); - u.ba.pC->nullRow = 0; - pIn2 = &aMem[pOp->p2]; - u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2); - u.ba.pC->rowidIsValid = 0; - u.ba.pC->deferredMoveto = 1; - } +case OP_Integer: { /* out2-prerelease */ + pOut->u.i = pOp->p1; break; } - -/* Opcode: Found P1 P2 P3 P4 * -** -** If P4==0 then register P3 holds a blob constructed by MakeRecord. If -** P4>0 then register P3 is the first of P4 registers that form an unpacked -** record. +/* Opcode: Int64 * P2 * P4 * ** -** Cursor P1 is on an index btree. If the record identified by P3 and P4 -** is a prefix of any entry in P1 then a jump is made to P2 and -** P1 is left pointing at the matching entry. +** P4 is a pointer to a 64-bit integer value. +** Write that value into register P2. */ -/* Opcode: NotFound P1 P2 P3 P4 * -** -** If P4==0 then register P3 holds a blob constructed by MakeRecord. If -** P4>0 then register P3 is the first of P4 registers that form an unpacked -** record. -** -** Cursor P1 is on an index btree. If the record identified by P3 and P4 -** is not the prefix of any entry in P1 then a jump is made to P2. If P1 -** does contain an entry whose prefix matches the P3/P4 record then control -** falls through to the next instruction and P1 is left pointing at the -** matching entry. +case OP_Int64: { /* out2-prerelease */ + assert( pOp->p4.pI64!=0 ); + pOut->u.i = *pOp->p4.pI64; + break; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* Opcode: Real * P2 * P4 * ** -** See also: Found, NotExists, IsUnique +** P4 is a pointer to a 64-bit floating point value. +** Write that value into register P2. */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ -#if 0 /* local variables moved into u.bb */ - int alreadyExists; - VdbeCursor *pC; - int res; - UnpackedRecord *pIdxKey; - UnpackedRecord r; - char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; -#endif /* local variables moved into u.bb */ - -#ifdef SQLITE_TEST - sqlite3_found_count++; +case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ + pOut->flags = MEM_Real; + assert( !sqlite3IsNaN(*pOp->p4.pReal) ); + pOut->r = *pOp->p4.pReal; + break; +} #endif - u.bb.alreadyExists = 0; - assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p4type==P4_INT32 ); - u.bb.pC = p->apCsr[pOp->p1]; - assert( u.bb.pC!=0 ); - pIn3 = &aMem[pOp->p3]; - if( ALWAYS(u.bb.pC->pCursor!=0) ){ +/* Opcode: String8 * P2 * P4 * +** +** P4 points to a nul terminated UTF-8 string. This opcode is transformed +** into an OP_String before it is executed for the first time. +*/ +case OP_String8: { /* same as TK_STRING, out2-prerelease */ + assert( pOp->p4.z!=0 ); + pOp->opcode = OP_String; + pOp->p1 = sqlite3Strlen30(pOp->p4.z); - assert( u.bb.pC->isTable==0 ); - if( pOp->p4.i>0 ){ - u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo; - u.bb.r.nField = (u16)pOp->p4.i; - u.bb.r.aMem = pIn3; -#ifdef SQLITE_DEBUG - { int i; for(i=0; iflags & MEM_Blob ); - assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ - u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z, - u.bb.aTempRec, sizeof(u.bb.aTempRec)); - if( u.bb.pIdxKey==0 ){ - goto no_mem; - } - u.bb.pIdxKey->flags |= UNPACKED_PREFIX_MATCH; - } - rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res); - if( pOp->p4.i==0 ){ - sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey); - } - if( rc!=SQLITE_OK ){ - break; +#ifndef SQLITE_OMIT_UTF16 + if( encoding!=SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); + if( rc==SQLITE_TOOBIG ) goto too_big; + if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; + assert( pOut->zMalloc==pOut->z ); + assert( pOut->flags & MEM_Dyn ); + pOut->zMalloc = 0; + pOut->flags |= MEM_Static; + pOut->flags &= ~MEM_Dyn; + if( pOp->p4type==P4_DYNAMIC ){ + sqlite3DbFree(db, pOp->p4.z); } - u.bb.alreadyExists = (u.bb.res==0); - u.bb.pC->deferredMoveto = 0; - u.bb.pC->cacheStatus = CACHE_STALE; + pOp->p4type = P4_DYNAMIC; + pOp->p4.z = pOut->z; + pOp->p1 = pOut->n; } - if( pOp->opcode==OP_Found ){ - if( u.bb.alreadyExists ) pc = pOp->p2 - 1; - }else{ - if( !u.bb.alreadyExists ) pc = pOp->p2 - 1; +#endif + if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; } + /* Fall through to the next case, OP_String */ +} + +/* Opcode: String P1 P2 * P4 * +** +** The string value P4 of length P1 (bytes) is stored in register P2. +*/ +case OP_String: { /* out2-prerelease */ + assert( pOp->p4.z!=0 ); + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = pOp->p4.z; + pOut->n = pOp->p1; + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); break; } -/* Opcode: IsUnique P1 P2 P3 P4 * -** -** Cursor P1 is open on an index b-tree - that is to say, a btree which -** no data and where the key are records generated by OP_MakeRecord with -** the list field being the integer ROWID of the entry that the index -** entry refers to. +/* Opcode: Null * P2 * * * ** -** The P3 register contains an integer record number. Call this record -** number R. Register P4 is the first in a set of N contiguous registers -** that make up an unpacked index key that can be used with cursor P1. -** The value of N can be inferred from the cursor. N includes the rowid -** value appended to the end of the index record. This rowid value may -** or may not be the same as R. +** Write a NULL into register P2. +*/ +case OP_Null: { /* out2-prerelease */ + pOut->flags = MEM_Null; + break; +} + + +/* Opcode: Blob P1 P2 * P4 ** -** If any of the N registers beginning with register P4 contains a NULL -** value, jump immediately to P2. +** P4 points to a blob of data P1 bytes long. Store this +** blob in register P2. +*/ +case OP_Blob: { /* out2-prerelease */ + assert( pOp->p1 <= SQLITE_MAX_LENGTH ); + sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Variable P1 P2 * P4 * ** -** Otherwise, this instruction checks if cursor P1 contains an entry -** where the first (N-1) fields match but the rowid value at the end -** of the index entry is not R. If there is no such entry, control jumps -** to instruction P2. Otherwise, the rowid of the conflicting index -** entry is copied to register P3 and control falls through to the next -** instruction. +** Transfer the values of bound parameter P1 into register P2 ** -** See also: NotFound, NotExists, Found +** If the parameter is named, then its name appears in P4 and P3==1. +** The P4 value is used by sqlite3_bind_parameter_name(). */ -case OP_IsUnique: { /* jump, in3 */ -#if 0 /* local variables moved into u.bc */ - u16 ii; - VdbeCursor *pCx; - BtCursor *pCrsr; - u16 nField; - Mem *aMx; - UnpackedRecord r; /* B-Tree index search key */ - i64 R; /* Rowid stored in register P3 */ -#endif /* local variables moved into u.bc */ - - pIn3 = &aMem[pOp->p3]; - u.bc.aMx = &aMem[pOp->p4.i]; - /* Assert that the values of parameters P1 and P4 are in range. */ - assert( pOp->p4type==P4_INT32 ); - assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem ); - assert( pOp->p1>=0 && pOp->p1nCursor ); - - /* Find the index cursor. */ - u.bc.pCx = p->apCsr[pOp->p1]; - assert( u.bc.pCx->deferredMoveto==0 ); - u.bc.pCx->seekResult = 0; - u.bc.pCx->cacheStatus = CACHE_STALE; - u.bc.pCrsr = u.bc.pCx->pCursor; +case OP_Variable: { /* out2-prerelease */ +#if 0 /* local variables moved into u.ab */ + Mem *pVar; /* Value being transferred */ +#endif /* local variables moved into u.ab */ - /* If any of the values are NULL, take the jump. */ - u.bc.nField = u.bc.pCx->pKeyInfo->nField; - for(u.bc.ii=0; u.bc.iip2 - 1; - u.bc.pCrsr = 0; - break; - } + assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); + u.ab.pVar = &p->aVar[pOp->p1 - 1]; + if( sqlite3VdbeMemTooBig(u.ab.pVar) ){ + goto too_big; } - assert( (u.bc.aMx[u.bc.nField].flags & MEM_Null)==0 ); + sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} - if( u.bc.pCrsr!=0 ){ - /* Populate the index search key. */ - u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo; - u.bc.r.nField = u.bc.nField + 1; - u.bc.r.flags = UNPACKED_PREFIX_SEARCH; - u.bc.r.aMem = u.bc.aMx; -#ifdef SQLITE_DEBUG - { int i; for(i=0; iu.i; + u.ac.n = pOp->p3; + u.ac.p1 = pOp->p1; + u.ac.p2 = pOp->p2; + assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 ); + assert( u.ac.p1+u.ac.n<=u.ac.p2 || u.ac.p2+u.ac.n<=u.ac.p1 ); - /* Search the B-Tree index. If no conflicting record is found, jump - ** to P2. Otherwise, copy the rowid of the conflicting record to - ** register P3 and fall through to the next instruction. */ - rc = sqlite3BtreeMovetoUnpacked(u.bc.pCrsr, &u.bc.r, 0, 0, &u.bc.pCx->seekResult); - if( (u.bc.r.flags & UNPACKED_PREFIX_SEARCH) || u.bc.r.rowid==u.bc.R ){ - pc = pOp->p2 - 1; - }else{ - pIn3->u.i = u.bc.r.rowid; - } + pIn1 = &aMem[u.ac.p1]; + pOut = &aMem[u.ac.p2]; + while( u.ac.n-- ){ + assert( pOut<=&aMem[p->nMem] ); + assert( pIn1<=&aMem[p->nMem] ); + assert( memIsValid(pIn1) ); + memAboutToChange(p, pOut); + u.ac.zMalloc = pOut->zMalloc; + pOut->zMalloc = 0; + sqlite3VdbeMemMove(pOut, pIn1); + pIn1->zMalloc = u.ac.zMalloc; + REGISTER_TRACE(u.ac.p2++, pOut); + pIn1++; + pOut++; } break; } -/* Opcode: NotExists P1 P2 P3 * * -** -** Use the content of register P3 as a integer key. If a record -** with that key does not exist in table of P1, then jump to P2. -** If the record does exist, then fall through. The cursor is left -** pointing to the record if it exists. +/* Opcode: Copy P1 P2 * * * ** -** The difference between this operation and NotFound is that this -** operation assumes the key is an integer and that P1 is a table whereas -** NotFound assumes key is a blob constructed from MakeRecord and -** P1 is an index. +** Make a copy of register P1 into register P2. ** -** See also: Found, NotFound, IsUnique +** This instruction makes a deep copy of the value. A duplicate +** is made of any string or blob constant. See also OP_SCopy. */ -case OP_NotExists: { /* jump, in3 */ -#if 0 /* local variables moved into u.bd */ - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - u64 iKey; -#endif /* local variables moved into u.bd */ - - pIn3 = &aMem[pOp->p3]; - assert( pIn3->flags & MEM_Int ); - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bd.pC = p->apCsr[pOp->p1]; - assert( u.bd.pC!=0 ); - assert( u.bd.pC->isTable ); - assert( u.bd.pC->pseudoTableReg==0 ); - u.bd.pCrsr = u.bd.pC->pCursor; - if( u.bd.pCrsr!=0 ){ - u.bd.res = 0; - u.bd.iKey = pIn3->u.i; - rc = sqlite3BtreeMovetoUnpacked(u.bd.pCrsr, 0, u.bd.iKey, 0, &u.bd.res); - u.bd.pC->lastRowid = pIn3->u.i; - u.bd.pC->rowidIsValid = u.bd.res==0 ?1:0; - u.bd.pC->nullRow = 0; - u.bd.pC->cacheStatus = CACHE_STALE; - u.bd.pC->deferredMoveto = 0; - if( u.bd.res!=0 ){ - pc = pOp->p2 - 1; - assert( u.bd.pC->rowidIsValid==0 ); - } - u.bd.pC->seekResult = u.bd.res; - }else{ - /* This happens when an attempt to open a read cursor on the - ** sqlite_master table returns SQLITE_EMPTY. - */ - pc = pOp->p2 - 1; - assert( u.bd.pC->rowidIsValid==0 ); - u.bd.pC->seekResult = 0; - } +case OP_Copy: { /* in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + assert( pOut!=pIn1 ); + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); + Deephemeralize(pOut); + REGISTER_TRACE(pOp->p2, pOut); break; } -/* Opcode: Sequence P1 P2 * * * +/* Opcode: SCopy P1 P2 * * * ** -** Find the next available sequence number for cursor P1. -** Write the sequence number into register P2. -** The sequence number on the cursor is incremented after this -** instruction. +** Make a shallow copy of register P1 into register P2. +** +** This instruction makes a shallow copy of the value. If the value +** is a string or blob, then the copy is only a pointer to the +** original and hence if the original changes so will the copy. +** Worse, if the original is deallocated, the copy becomes invalid. +** Thus the program must guarantee that the original will not change +** during the lifetime of the copy. Use OP_Copy to make a complete +** copy. */ -case OP_Sequence: { /* out2-prerelease */ - assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( p->apCsr[pOp->p1]!=0 ); - pOut->u.i = p->apCsr[pOp->p1]->seqCount++; +case OP_SCopy: { /* in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + assert( pOut!=pIn1 ); + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); +#ifdef SQLITE_DEBUG + if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; +#endif + REGISTER_TRACE(pOp->p2, pOut); break; } - -/* Opcode: NewRowid P1 P2 P3 * * -** -** Get a new integer record number (a.k.a "rowid") used as the key to a table. -** The record number is not previously used as a key in the database -** table that cursor P1 points to. The new record number is written -** written to register P2. +/* Opcode: ResultRow P1 P2 * * * ** -** If P3>0 then P3 is a register in the root frame of this VDBE that holds -** the largest previously generated record number. No new record numbers are -** allowed to be less than this value. When this value reaches its maximum, -** a SQLITE_FULL error is generated. The P3 register is updated with the ' -** generated record number. This P3 mechanism is used to help implement the -** AUTOINCREMENT feature. +** The registers P1 through P1+P2-1 contain a single row of +** results. This opcode causes the sqlite3_step() call to terminate +** with an SQLITE_ROW return code and it sets up the sqlite3_stmt +** structure to provide access to the top P1 values as the result +** row. */ -case OP_NewRowid: { /* out2-prerelease */ -#if 0 /* local variables moved into u.be */ - i64 v; /* The new rowid */ - VdbeCursor *pC; /* Cursor of table to get the new rowid */ - int res; /* Result of an sqlite3BtreeLast() */ - int cnt; /* Counter to limit the number of searches */ - Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ - VdbeFrame *pFrame; /* Root frame of VDBE */ -#endif /* local variables moved into u.be */ - - u.be.v = 0; - u.be.res = 0; - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.be.pC = p->apCsr[pOp->p1]; - assert( u.be.pC!=0 ); - if( NEVER(u.be.pC->pCursor==0) ){ - /* The zero initialization above is all that is needed */ - }else{ - /* The next rowid or record number (different terms for the same - ** thing) is obtained in a two-step algorithm. - ** - ** First we attempt to find the largest existing rowid and add one - ** to that. But if the largest existing rowid is already the maximum - ** positive integer, we have to fall through to the second - ** probabilistic algorithm - ** - ** The second algorithm is to select a rowid at random and see if - ** it already exists in the table. If it does not exist, we have - ** succeeded. If the random rowid does exist, we select a new one - ** and try again, up to 100 times. - */ - assert( u.be.pC->isTable ); - u.be.cnt = 0; +case OP_ResultRow: { +#if 0 /* local variables moved into u.ad */ + Mem *pMem; + int i; +#endif /* local variables moved into u.ad */ + assert( p->nResColumn==pOp->p2 ); + assert( pOp->p1>0 ); + assert( pOp->p1+pOp->p2<=p->nMem+1 ); -#ifdef SQLITE_32BIT_ROWID -# define MAX_ROWID 0x7fffffff -#else - /* Some compilers complain about constants of the form 0x7fffffffffffffff. - ** Others complain about 0x7ffffffffffffffffLL. The following macro seems - ** to provide the constant while making all compilers happy. - */ -# define MAX_ROWID (i64)( (((u64)0x7fffffff)<<32) | (u64)0xffffffff ) -#endif + /* If this statement has violated immediate foreign key constraints, do + ** not return the number of rows modified. And do not RELEASE the statement + ** transaction. It needs to be rolled back. */ + if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ + assert( db->flags&SQLITE_CountRows ); + assert( p->usesStmtJournal ); + break; + } - if( !u.be.pC->useRandomRowid ){ - u.be.v = sqlite3BtreeGetCachedRowid(u.be.pC->pCursor); - if( u.be.v==0 ){ - rc = sqlite3BtreeLast(u.be.pC->pCursor, &u.be.res); - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - if( u.be.res ){ - u.be.v = 1; /* IMP: R-61914-48074 */ - }else{ - assert( sqlite3BtreeCursorIsValid(u.be.pC->pCursor) ); - rc = sqlite3BtreeKeySize(u.be.pC->pCursor, &u.be.v); - assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ - if( u.be.v==MAX_ROWID ){ - u.be.pC->useRandomRowid = 1; - }else{ - u.be.v++; /* IMP: R-29538-34987 */ - } - } - } + /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then + ** DML statements invoke this opcode to return the number of rows + ** modified to the user. This is the only way that a VM that + ** opens a statement transaction may invoke this opcode. + ** + ** In case this is such a statement, close any statement transaction + ** opened by this VM before returning control to the user. This is to + ** ensure that statement-transactions are always nested, not overlapping. + ** If the open statement-transaction is not closed here, then the user + ** may step another VM that opens its own statement transaction. This + ** may lead to overlapping statement transactions. + ** + ** The statement transaction is never a top-level transaction. Hence + ** the RELEASE call below can never fail. + */ + assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); + rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); + if( NEVER(rc!=SQLITE_OK) ){ + break; + } -#ifndef SQLITE_OMIT_AUTOINCREMENT - if( pOp->p3 ){ - /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3>0 ); - if( p->pFrame ){ - for(u.be.pFrame=p->pFrame; u.be.pFrame->pParent; u.be.pFrame=u.be.pFrame->pParent); - /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=u.be.pFrame->nMem ); - u.be.pMem = &u.be.pFrame->aMem[pOp->p3]; - }else{ - /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=p->nMem ); - u.be.pMem = &aMem[pOp->p3]; - memAboutToChange(p, u.be.pMem); - } - assert( memIsValid(u.be.pMem) ); + /* Invalidate all ephemeral cursor row caches */ + p->cacheCtr = (p->cacheCtr + 2)|1; - REGISTER_TRACE(pOp->p3, u.be.pMem); - sqlite3VdbeMemIntegerify(u.be.pMem); - assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ - if( u.be.pMem->u.i==MAX_ROWID || u.be.pC->useRandomRowid ){ - rc = SQLITE_FULL; /* IMP: R-12275-61338 */ - goto abort_due_to_error; - } - if( u.be.vu.i+1 ){ - u.be.v = u.be.pMem->u.i + 1; - } - u.be.pMem->u.i = u.be.v; - } -#endif + /* Make sure the results of the current row are \000 terminated + ** and have an assigned type. The results are de-ephemeralized as + ** as side effect. + */ + u.ad.pMem = p->pResultSet = &aMem[pOp->p1]; + for(u.ad.i=0; u.ad.ip2; u.ad.i++){ + assert( memIsValid(&u.ad.pMem[u.ad.i]) ); + Deephemeralize(&u.ad.pMem[u.ad.i]); + assert( (u.ad.pMem[u.ad.i].flags & MEM_Ephem)==0 + || (u.ad.pMem[u.ad.i].flags & (MEM_Str|MEM_Blob))==0 ); + sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]); + sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]); + REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]); + } + if( db->mallocFailed ) goto no_mem; - sqlite3BtreeSetCachedRowid(u.be.pC->pCursor, u.be.vuseRandomRowid ){ - /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the - ** largest possible integer (9223372036854775807) then the database - ** engine starts picking positive candidate ROWIDs at random until - ** it finds one that is not previously used. */ - assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is - ** an AUTOINCREMENT table. */ - /* on the first attempt, simply do one more than previous */ - u.be.v = db->lastRowid; - u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - u.be.v++; /* ensure non-zero */ - u.be.cnt = 0; - while( ((rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, - 0, &u.be.res))==SQLITE_OK) - && (u.be.res==0) - && (++u.be.cnt<100)){ - /* collision - try another random rowid */ - sqlite3_randomness(sizeof(u.be.v), &u.be.v); - if( u.be.cnt<5 ){ - /* try "small" random rowids for the initial attempts */ - u.be.v &= 0xffffff; - }else{ - u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - } - u.be.v++; /* ensure non-zero */ - } - if( rc==SQLITE_OK && u.be.res==0 ){ - rc = SQLITE_FULL; /* IMP: R-38219-53002 */ - goto abort_due_to_error; - } - assert( u.be.v>0 ); /* EV: R-40812-03570 */ - } - u.be.pC->rowidIsValid = 0; - u.be.pC->deferredMoveto = 0; - u.be.pC->cacheStatus = CACHE_STALE; - } - pOut->u.i = u.be.v; - break; + /* Return SQLITE_ROW + */ + p->pc = pc + 1; + rc = SQLITE_ROW; + goto vdbe_return; } -/* Opcode: Insert P1 P2 P3 P4 P5 -** -** Write an entry into the table of cursor P1. A new entry is -** created if it doesn't already exist or the data for an existing -** entry is overwritten. The data is the value MEM_Blob stored in register -** number P2. The key is stored in register P3. The key must -** be a MEM_Int. -** -** If the OPFLAG_NCHANGE flag of P5 is set, then the row change count is -** incremented (otherwise not). If the OPFLAG_LASTROWID flag of P5 is set, -** then rowid is stored for subsequent return by the -** sqlite3_last_insert_rowid() function (otherwise it is unmodified). -** -** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of -** the last seek operation (OP_NotExists) was a success, then this -** operation will not attempt to find the appropriate row before doing -** the insert but will instead overwrite the row that the cursor is -** currently pointing to. Presumably, the prior OP_NotExists opcode -** has already positioned the cursor correctly. This is an optimization -** that boosts performance by avoiding redundant seeks. -** -** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an -** UPDATE operation. Otherwise (if the flag is clear) then this opcode -** is part of an INSERT operation. The difference is only important to -** the update hook. -** -** Parameter P4 may point to a string containing the table-name, or -** may be NULL. If it is not NULL, then the update-hook -** (sqlite3.xUpdateCallback) is invoked following a successful insert. +/* Opcode: Concat P1 P2 P3 * * ** -** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically -** allocated, then ownership of P2 is transferred to the pseudo-cursor -** and register P2 becomes ephemeral. If the cursor is changed, the -** value of register P2 will then change. Make sure this does not -** cause any problems.) +** Add the text in register P1 onto the end of the text in +** register P2 and store the result in register P3. +** If either the P1 or P2 text are NULL then store NULL in P3. ** -** This instruction only works on tables. The equivalent instruction -** for indices is OP_IdxInsert. -*/ -/* Opcode: InsertInt P1 P2 P3 P4 P5 +** P3 = P2 || P1 ** -** This works exactly like OP_Insert except that the key is the -** integer value P3, not the value of the integer stored in register P3. +** It is illegal for P1 and P3 to be the same register. Sometimes, +** if P3 is the same register as P2, the implementation is able +** to avoid a memcpy(). */ -case OP_Insert: -case OP_InsertInt: { -#if 0 /* local variables moved into u.bf */ - Mem *pData; /* MEM cell holding data for the record to be inserted */ - Mem *pKey; /* MEM cell holding key for the record */ - i64 iKey; /* The integer ROWID or key for the record to be inserted */ - VdbeCursor *pC; /* Cursor to table into which insert is written */ - int nZero; /* Number of zero-bytes to append */ - int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ - const char *zDb; /* database name - used by the update hook */ - const char *zTbl; /* Table name - used by the opdate hook */ - int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ -#endif /* local variables moved into u.bf */ - - u.bf.pData = &aMem[pOp->p2]; - assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( memIsValid(u.bf.pData) ); - u.bf.pC = p->apCsr[pOp->p1]; - assert( u.bf.pC!=0 ); - assert( u.bf.pC->pCursor!=0 ); - assert( u.bf.pC->pseudoTableReg==0 ); - assert( u.bf.pC->isTable ); - REGISTER_TRACE(pOp->p2, u.bf.pData); +case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ +#if 0 /* local variables moved into u.ae */ + i64 nByte; +#endif /* local variables moved into u.ae */ - if( pOp->opcode==OP_Insert ){ - u.bf.pKey = &aMem[pOp->p3]; - assert( u.bf.pKey->flags & MEM_Int ); - assert( memIsValid(u.bf.pKey) ); - REGISTER_TRACE(pOp->p3, u.bf.pKey); - u.bf.iKey = u.bf.pKey->u.i; - }else{ - assert( pOp->opcode==OP_InsertInt ); - u.bf.iKey = pOp->p3; + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; + assert( pIn1!=pOut ); + if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + break; } - - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = u.bf.iKey; - if( u.bf.pData->flags & MEM_Null ){ - u.bf.pData->z = 0; - u.bf.pData->n = 0; - }else{ - assert( u.bf.pData->flags & (MEM_Blob|MEM_Str) ); + if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; + Stringify(pIn1, encoding); + Stringify(pIn2, encoding); + u.ae.nByte = pIn1->n + pIn2->n; + if( u.ae.nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; } - u.bf.seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? u.bf.pC->seekResult : 0); - if( u.bf.pData->flags & MEM_Zero ){ - u.bf.nZero = u.bf.pData->u.nZero; - }else{ - u.bf.nZero = 0; + MemSetTypeFlag(pOut, MEM_Str); + if( sqlite3VdbeMemGrow(pOut, (int)u.ae.nByte+2, pOut==pIn2) ){ + goto no_mem; } - sqlite3BtreeSetCachedRowid(u.bf.pC->pCursor, 0); - rc = sqlite3BtreeInsert(u.bf.pC->pCursor, 0, u.bf.iKey, - u.bf.pData->z, u.bf.pData->n, u.bf.nZero, - pOp->p5 & OPFLAG_APPEND, u.bf.seekResult - ); - u.bf.pC->rowidIsValid = 0; - u.bf.pC->deferredMoveto = 0; - u.bf.pC->cacheStatus = CACHE_STALE; - - /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ - u.bf.zDb = db->aDb[u.bf.pC->iDb].zName; - u.bf.zTbl = pOp->p4.z; - u.bf.op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); - assert( u.bf.pC->isTable ); - db->xUpdateCallback(db->pUpdateArg, u.bf.op, u.bf.zDb, u.bf.zTbl, u.bf.iKey); - assert( u.bf.pC->iDb>=0 ); + if( pOut!=pIn2 ){ + memcpy(pOut->z, pIn2->z, pIn2->n); } + memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + pOut->z[u.ae.nByte] = 0; + pOut->z[u.ae.nByte+1] = 0; + pOut->flags |= MEM_Term; + pOut->n = (int)u.ae.nByte; + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); break; } -/* Opcode: Delete P1 P2 * P4 * +/* Opcode: Add P1 P2 P3 * * ** -** Delete the record at which the P1 cursor is currently pointing. +** Add the value in register P1 to the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Multiply P1 P2 P3 * * ** -** The cursor will be left pointing at either the next or the previous -** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within an Next loop. ** -** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is -** incremented (otherwise not). +** Multiply the value in register P1 by the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Subtract P1 P2 P3 * * ** -** P1 must not be pseudo-table. It has to be a real table with -** multiple rows. +** Subtract the value in register P1 from the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Divide P1 P2 P3 * * ** -** If P4 is not NULL, then it is the name of the table that P1 is -** pointing to. The update hook will be invoked, if it exists. -** If P4 is not NULL then the P1 cursor must have been positioned -** using OP_NotFound prior to invoking this opcode. +** Divide the value in register P1 by the value in register P2 +** and store the result in register P3 (P3=P2/P1). If the value in +** register P1 is zero, then the result is NULL. If either input is +** NULL, the result is NULL. */ -case OP_Delete: { -#if 0 /* local variables moved into u.bg */ - i64 iKey; - VdbeCursor *pC; -#endif /* local variables moved into u.bg */ - - u.bg.iKey = 0; - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bg.pC = p->apCsr[pOp->p1]; - assert( u.bg.pC!=0 ); - assert( u.bg.pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ +/* Opcode: Remainder P1 P2 P3 * * +** +** Compute the remainder after integer division of the value in +** register P1 by the value in register P2 and store the result in P3. +** If the value in register P2 is zero the result is NULL. +** If either operand is NULL, the result is NULL. +*/ +case OP_Add: /* same as TK_PLUS, in1, in2, out3 */ +case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ +case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ +case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ +case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ +#if 0 /* local variables moved into u.af */ + int flags; /* Combined MEM_* flags from both inputs */ + i64 iA; /* Integer value of left operand */ + i64 iB; /* Integer value of right operand */ + double rA; /* Real value of left operand */ + double rB; /* Real value of right operand */ +#endif /* local variables moved into u.af */ - /* If the update-hook will be invoked, set u.bg.iKey to the rowid of the - ** row being deleted. - */ - if( db->xUpdateCallback && pOp->p4.z ){ - assert( u.bg.pC->isTable ); - assert( u.bg.pC->rowidIsValid ); /* lastRowid set by previous OP_NotFound */ - u.bg.iKey = u.bg.pC->lastRowid; + pIn1 = &aMem[pOp->p1]; + applyNumericAffinity(pIn1); + pIn2 = &aMem[pOp->p2]; + applyNumericAffinity(pIn2); + pOut = &aMem[pOp->p3]; + u.af.flags = pIn1->flags | pIn2->flags; + if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; + if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ + u.af.iA = pIn1->u.i; + u.af.iB = pIn2->u.i; + switch( pOp->opcode ){ + case OP_Add: if( sqlite3AddInt64(&u.af.iB,u.af.iA) ) goto fp_math; break; + case OP_Subtract: if( sqlite3SubInt64(&u.af.iB,u.af.iA) ) goto fp_math; break; + case OP_Multiply: if( sqlite3MulInt64(&u.af.iB,u.af.iA) ) goto fp_math; break; + case OP_Divide: { + if( u.af.iA==0 ) goto arithmetic_result_is_null; + if( u.af.iA==-1 && u.af.iB==SMALLEST_INT64 ) goto fp_math; + u.af.iB /= u.af.iA; + break; + } + default: { + if( u.af.iA==0 ) goto arithmetic_result_is_null; + if( u.af.iA==-1 ) u.af.iA = 1; + u.af.iB %= u.af.iA; + break; + } + } + pOut->u.i = u.af.iB; + MemSetTypeFlag(pOut, MEM_Int); + }else{ +fp_math: + u.af.rA = sqlite3VdbeRealValue(pIn1); + u.af.rB = sqlite3VdbeRealValue(pIn2); + switch( pOp->opcode ){ + case OP_Add: u.af.rB += u.af.rA; break; + case OP_Subtract: u.af.rB -= u.af.rA; break; + case OP_Multiply: u.af.rB *= u.af.rA; break; + case OP_Divide: { + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + if( u.af.rA==(double)0 ) goto arithmetic_result_is_null; + u.af.rB /= u.af.rA; + break; + } + default: { + u.af.iA = (i64)u.af.rA; + u.af.iB = (i64)u.af.rB; + if( u.af.iA==0 ) goto arithmetic_result_is_null; + if( u.af.iA==-1 ) u.af.iA = 1; + u.af.rB = (double)(u.af.iB % u.af.iA); + break; + } + } +#ifdef SQLITE_OMIT_FLOATING_POINT + pOut->u.i = u.af.rB; + MemSetTypeFlag(pOut, MEM_Int); +#else + if( sqlite3IsNaN(u.af.rB) ){ + goto arithmetic_result_is_null; + } + pOut->r = u.af.rB; + MemSetTypeFlag(pOut, MEM_Real); + if( (u.af.flags & MEM_Real)==0 ){ + sqlite3VdbeIntegerAffinity(pOut); + } +#endif } + break; - /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or - ** OP_Column on the same table without any intervening operations that - ** might move or invalidate the cursor. Hence cursor u.bg.pC is always pointing - ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation - ** below is always a no-op and cannot fail. We will run it anyhow, though, - ** to guard against future changes to the code generator. - **/ - assert( u.bg.pC->deferredMoveto==0 ); - rc = sqlite3VdbeCursorMoveto(u.bg.pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; - - sqlite3BtreeSetCachedRowid(u.bg.pC->pCursor, 0); - rc = sqlite3BtreeDelete(u.bg.pC->pCursor); - u.bg.pC->cacheStatus = CACHE_STALE; - - /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ - const char *zDb = db->aDb[u.bg.pC->iDb].zName; - const char *zTbl = pOp->p4.z; - db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, zTbl, u.bg.iKey); - assert( u.bg.pC->iDb>=0 ); - } - if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; +arithmetic_result_is_null: + sqlite3VdbeMemSetNull(pOut); break; } -/* Opcode: ResetCount * * * * * + +/* Opcode: CollSeq * * P4 ** -** The value of the change counter is copied to the database handle -** change counter (returned by subsequent calls to sqlite3_changes()). -** Then the VMs internal change counter resets to 0. -** This is used by trigger programs. +** P4 is a pointer to a CollSeq struct. If the next call to a user function +** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will +** be returned. This is used by the built-in min(), max() and nullif() +** functions. +** +** The interface used by the implementation of the aforementioned functions +** to retrieve the collation sequence set by this opcode is not available +** publicly, only to user functions defined in func.c. */ -case OP_ResetCount: { - sqlite3VdbeSetChanges(db, p->nChange); - p->nChange = 0; +case OP_CollSeq: { + assert( pOp->p4type==P4_COLLSEQ ); break; } -/* Opcode: RowData P1 P2 * * * -** -** Write into register P2 the complete row data for cursor P1. -** There is no interpretation of the data. -** It is just copied onto the P2 register exactly as -** it is found in the database file. +/* Opcode: Function P1 P2 P3 P4 P5 ** -** If the P1 cursor must be pointing to a valid row (not a NULL row) -** of a real table, not a pseudo-table. -*/ -/* Opcode: RowKey P1 P2 * * * +** Invoke a user function (P4 is a pointer to a Function structure that +** defines the function) with P5 arguments taken from register P2 and +** successors. The result of the function is stored in register P3. +** Register P3 must not be one of the function inputs. ** -** Write into register P2 the complete row key for cursor P1. -** There is no interpretation of the data. -** The key is copied onto the P3 register exactly as -** it is found in the database file. +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. ** -** If the P1 cursor must be pointing to a valid row (not a NULL row) -** of a real table, not a pseudo-table. +** See also: AggStep and AggFinal */ -case OP_RowKey: -case OP_RowData: { -#if 0 /* local variables moved into u.bh */ - VdbeCursor *pC; - BtCursor *pCrsr; - u32 n; - i64 n64; -#endif /* local variables moved into u.bh */ +case OP_Function: { +#if 0 /* local variables moved into u.ag */ + int i; + Mem *pArg; + sqlite3_context ctx; + sqlite3_value **apVal; + int n; +#endif /* local variables moved into u.ag */ - pOut = &aMem[pOp->p2]; + u.ag.n = pOp->p5; + u.ag.apVal = p->apArg; + assert( u.ag.apVal || u.ag.n==0 ); + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + pOut = &aMem[pOp->p3]; memAboutToChange(p, pOut); - /* Note that RowKey and RowData are really exactly the same instruction */ - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bh.pC = p->apCsr[pOp->p1]; - assert( u.bh.pC->isTable || pOp->opcode==OP_RowKey ); - assert( u.bh.pC->isIndex || pOp->opcode==OP_RowData ); - assert( u.bh.pC!=0 ); - assert( u.bh.pC->nullRow==0 ); - assert( u.bh.pC->pseudoTableReg==0 ); - assert( u.bh.pC->pCursor!=0 ); - u.bh.pCrsr = u.bh.pC->pCursor; - assert( sqlite3BtreeCursorIsValid(u.bh.pCrsr) ); + assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+u.ag.n ); + u.ag.pArg = &aMem[pOp->p2]; + for(u.ag.i=0; u.ag.ip2+u.ag.i, u.ag.pArg); + } - /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or - ** OP_Rewind/Op_Next with no intervening instructions that might invalidate - ** the cursor. Hence the following sqlite3VdbeCursorMoveto() call is always - ** a no-op and can never fail. But we leave it in place as a safety. + assert( pOp->p4type==P4_FUNCDEF || pOp->p4type==P4_VDBEFUNC ); + if( pOp->p4type==P4_FUNCDEF ){ + u.ag.ctx.pFunc = pOp->p4.pFunc; + u.ag.ctx.pVdbeFunc = 0; + }else{ + u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc; + u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc; + } + + u.ag.ctx.s.flags = MEM_Null; + u.ag.ctx.s.db = db; + u.ag.ctx.s.xDel = 0; + u.ag.ctx.s.zMalloc = 0; + + /* The output cell may already have a buffer allocated. Move + ** the pointer to u.ag.ctx.s so in case the user-function can use + ** the already allocated buffer instead of allocating a new one. */ - assert( u.bh.pC->deferredMoveto==0 ); - rc = sqlite3VdbeCursorMoveto(u.bh.pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + sqlite3VdbeMemMove(&u.ag.ctx.s, pOut); + MemSetTypeFlag(&u.ag.ctx.s, MEM_Null); - if( u.bh.pC->isIndex ){ - assert( !u.bh.pC->isTable ); - rc = sqlite3BtreeKeySize(u.bh.pCrsr, &u.bh.n64); - assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ - if( u.bh.n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } - u.bh.n = (u32)u.bh.n64; - }else{ - rc = sqlite3BtreeDataSize(u.bh.pCrsr, &u.bh.n); - assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ - if( u.bh.n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } + u.ag.ctx.isError = 0; + if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ + assert( pOp>aOp ); + assert( pOp[-1].p4type==P4_COLLSEQ ); + assert( pOp[-1].opcode==OP_CollSeq ); + u.ag.ctx.pColl = pOp[-1].p4.pColl; } - if( sqlite3VdbeMemGrow(pOut, u.bh.n, 0) ){ + db->lastRowid = lastRowid; + (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; + + /* If any auxiliary data functions have been called by this user function, + ** immediately call the destructor for any non-static values. + */ + if( u.ag.ctx.pVdbeFunc ){ + sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1); + pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc; + pOp->p4type = P4_VDBEFUNC; + } + + if( db->mallocFailed ){ + /* Even though a malloc() has failed, the implementation of the + ** user function may have called an sqlite3_result_XXX() function + ** to return a value. The following call releases any resources + ** associated with such a value. + */ + sqlite3VdbeMemRelease(&u.ag.ctx.s); goto no_mem; } - pOut->n = u.bh.n; - MemSetTypeFlag(pOut, MEM_Blob); - if( u.bh.pC->isIndex ){ - rc = sqlite3BtreeKey(u.bh.pCrsr, 0, u.bh.n, pOut->z); - }else{ - rc = sqlite3BtreeData(u.bh.pCrsr, 0, u.bh.n, pOut->z); + + /* If the function returned an error, throw an exception */ + if( u.ag.ctx.isError ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s)); + rc = u.ag.ctx.isError; } - pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ - UPDATE_MAX_BLOBSIZE(pOut); - break; -} -/* Opcode: Rowid P1 P2 * * * -** -** Store in register P2 an integer which is the key of the table entry that -** P1 is currently point to. -** -** P1 can be either an ordinary table or a virtual table. There used to -** be a separate OP_VRowid opcode for use with virtual tables, but this -** one opcode now works for both table types. + /* Copy the result of the function into register P3 */ + sqlite3VdbeChangeEncoding(&u.ag.ctx.s, encoding); + sqlite3VdbeMemMove(pOut, &u.ag.ctx.s); + if( sqlite3VdbeMemTooBig(pOut) ){ + goto too_big; + } + +#if 0 + /* The app-defined function has done something that as caused this + ** statement to expire. (Perhaps the function called sqlite3_exec() + ** with a CREATE TABLE statement.) + */ + if( p->expired ) rc = SQLITE_ABORT; +#endif + + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: BitAnd P1 P2 P3 * * +** +** Take the bit-wise AND of the values in register P1 and P2 and +** store the result in register P3. +** If either input is NULL, the result is NULL. */ -case OP_Rowid: { /* out2-prerelease */ -#if 0 /* local variables moved into u.bi */ - VdbeCursor *pC; - i64 v; - sqlite3_vtab *pVtab; - const sqlite3_module *pModule; -#endif /* local variables moved into u.bi */ +/* Opcode: BitOr P1 P2 P3 * * +** +** Take the bit-wise OR of the values in register P1 and P2 and +** store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: ShiftLeft P1 P2 P3 * * +** +** Shift the integer value in register P2 to the left by the +** number of bits specified by the integer in register P1. +** Store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: ShiftRight P1 P2 P3 * * +** +** Shift the integer value in register P2 to the right by the +** number of bits specified by the integer in register P1. +** Store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +case OP_BitAnd: /* same as TK_BITAND, in1, in2, out3 */ +case OP_BitOr: /* same as TK_BITOR, in1, in2, out3 */ +case OP_ShiftLeft: /* same as TK_LSHIFT, in1, in2, out3 */ +case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ +#if 0 /* local variables moved into u.ah */ + i64 iA; + u64 uA; + i64 iB; + u8 op; +#endif /* local variables moved into u.ah */ - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bi.pC = p->apCsr[pOp->p1]; - assert( u.bi.pC!=0 ); - assert( u.bi.pC->pseudoTableReg==0 ); - if( u.bi.pC->nullRow ){ - pOut->flags = MEM_Null; + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; + if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); break; - }else if( u.bi.pC->deferredMoveto ){ - u.bi.v = u.bi.pC->movetoTarget; -#ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( u.bi.pC->pVtabCursor ){ - u.bi.pVtab = u.bi.pC->pVtabCursor->pVtab; - u.bi.pModule = u.bi.pVtab->pModule; - assert( u.bi.pModule->xRowid ); - rc = u.bi.pModule->xRowid(u.bi.pC->pVtabCursor, &u.bi.v); - importVtabErrMsg(p, u.bi.pVtab); -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - }else{ - assert( u.bi.pC->pCursor!=0 ); - rc = sqlite3VdbeCursorMoveto(u.bi.pC); - if( rc ) goto abort_due_to_error; - if( u.bi.pC->rowidIsValid ){ - u.bi.v = u.bi.pC->lastRowid; + } + u.ah.iA = sqlite3VdbeIntValue(pIn2); + u.ah.iB = sqlite3VdbeIntValue(pIn1); + u.ah.op = pOp->opcode; + if( u.ah.op==OP_BitAnd ){ + u.ah.iA &= u.ah.iB; + }else if( u.ah.op==OP_BitOr ){ + u.ah.iA |= u.ah.iB; + }else if( u.ah.iB!=0 ){ + assert( u.ah.op==OP_ShiftRight || u.ah.op==OP_ShiftLeft ); + + /* If shifting by a negative amount, shift in the other direction */ + if( u.ah.iB<0 ){ + assert( OP_ShiftRight==OP_ShiftLeft+1 ); + u.ah.op = 2*OP_ShiftLeft + 1 - u.ah.op; + u.ah.iB = u.ah.iB>(-64) ? -u.ah.iB : 64; + } + + if( u.ah.iB>=64 ){ + u.ah.iA = (u.ah.iA>=0 || u.ah.op==OP_ShiftLeft) ? 0 : -1; }else{ - rc = sqlite3BtreeKeySize(u.bi.pC->pCursor, &u.bi.v); - assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ + memcpy(&u.ah.uA, &u.ah.iA, sizeof(u.ah.uA)); + if( u.ah.op==OP_ShiftLeft ){ + u.ah.uA <<= u.ah.iB; + }else{ + u.ah.uA >>= u.ah.iB; + /* Sign-extend on a right shift of a negative number */ + if( u.ah.iA<0 ) u.ah.uA |= ((((u64)0xffffffff)<<32)|0xffffffff) << (64-u.ah.iB); + } + memcpy(&u.ah.iA, &u.ah.uA, sizeof(u.ah.iA)); } } - pOut->u.i = u.bi.v; + pOut->u.i = u.ah.iA; + MemSetTypeFlag(pOut, MEM_Int); break; } -/* Opcode: NullRow P1 * * * * +/* Opcode: AddImm P1 P2 * * * +** +** Add the constant P2 to the value in register P1. +** The result is always an integer. ** -** Move the cursor P1 to a null row. Any OP_Column operations -** that occur while the cursor is on the null row will always -** write a NULL. +** To force any register to be an integer, just add 0. */ -case OP_NullRow: { -#if 0 /* local variables moved into u.bj */ - VdbeCursor *pC; -#endif /* local variables moved into u.bj */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bj.pC = p->apCsr[pOp->p1]; - assert( u.bj.pC!=0 ); - u.bj.pC->nullRow = 1; - u.bj.pC->rowidIsValid = 0; - if( u.bj.pC->pCursor ){ - sqlite3BtreeClearCursor(u.bj.pC->pCursor); - } +case OP_AddImm: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + sqlite3VdbeMemIntegerify(pIn1); + pIn1->u.i += pOp->p2; break; } -/* Opcode: Last P1 P2 * * * -** -** The next use of the Rowid or Column or Next instruction for P1 -** will refer to the last entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +/* Opcode: MustBeInt P1 P2 * * * +** +** Force the value in register P1 to be an integer. If the value +** in P1 is not an integer and cannot be converted into an integer +** without data loss, then jump immediately to P2, or if P2==0 +** raise an SQLITE_MISMATCH exception. */ -case OP_Last: { /* jump */ -#if 0 /* local variables moved into u.bk */ - VdbeCursor *pC; - BtCursor *pCrsr; - int res; -#endif /* local variables moved into u.bk */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bk.pC = p->apCsr[pOp->p1]; - assert( u.bk.pC!=0 ); - u.bk.pCrsr = u.bk.pC->pCursor; - if( u.bk.pCrsr==0 ){ - u.bk.res = 1; +case OP_MustBeInt: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); + if( (pIn1->flags & MEM_Int)==0 ){ + if( pOp->p2==0 ){ + rc = SQLITE_MISMATCH; + goto abort_due_to_error; + }else{ + pc = pOp->p2 - 1; + } }else{ - rc = sqlite3BtreeLast(u.bk.pCrsr, &u.bk.res); - } - u.bk.pC->nullRow = (u8)u.bk.res; - u.bk.pC->deferredMoveto = 0; - u.bk.pC->rowidIsValid = 0; - u.bk.pC->cacheStatus = CACHE_STALE; - if( pOp->p2>0 && u.bk.res ){ - pc = pOp->p2 - 1; + MemSetTypeFlag(pIn1, MEM_Int); } break; } - -/* Opcode: Sort P1 P2 * * * -** -** This opcode does exactly the same thing as OP_Rewind except that -** it increments an undocumented global variable used for testing. +#ifndef SQLITE_OMIT_FLOATING_POINT +/* Opcode: RealAffinity P1 * * * * ** -** Sorting is accomplished by writing records into a sorting index, -** then rewinding that index and playing it back from beginning to -** end. We use the OP_Sort opcode instead of OP_Rewind to do the -** rewinding so that the global variable will be incremented and -** regression tests can determine whether or not the optimizer is -** correctly optimizing out sorts. -*/ -case OP_Sort: { /* jump */ -#ifdef SQLITE_TEST - sqlite3_sort_count++; - sqlite3_search_count--; -#endif - p->aCounter[SQLITE_STMTSTATUS_SORT-1]++; - /* Fall through into OP_Rewind */ -} -/* Opcode: Rewind P1 P2 * * * +** If register P1 holds an integer convert it to a real value. ** -** The next use of the Rowid or Column or Next instruction for P1 -** will refer to the first entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +** This opcode is used when extracting information from a column that +** has REAL affinity. Such column values may still be stored as +** integers, for space efficiency, but after extraction we want them +** to have only a real value. */ -case OP_Rewind: { /* jump */ -#if 0 /* local variables moved into u.bl */ - VdbeCursor *pC; - BtCursor *pCrsr; - int res; -#endif /* local variables moved into u.bl */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bl.pC = p->apCsr[pOp->p1]; - assert( u.bl.pC!=0 ); - u.bl.res = 1; - if( (u.bl.pCrsr = u.bl.pC->pCursor)!=0 ){ - rc = sqlite3BtreeFirst(u.bl.pCrsr, &u.bl.res); - u.bl.pC->atFirst = u.bl.res==0 ?1:0; - u.bl.pC->deferredMoveto = 0; - u.bl.pC->cacheStatus = CACHE_STALE; - u.bl.pC->rowidIsValid = 0; - } - u.bl.pC->nullRow = (u8)u.bl.res; - assert( pOp->p2>0 && pOp->p2nOp ); - if( u.bl.res ){ - pc = pOp->p2 - 1; +case OP_RealAffinity: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Int ){ + sqlite3VdbeMemRealify(pIn1); } break; } +#endif -/* Opcode: Next P1 P2 * * P5 -** -** Advance cursor P1 so that it points to the next key/data pair in its -** table or index. If there are no more key/value pairs then fall through -** to the following instruction. But if the cursor advance was successful, -** jump immediately to P2. -** -** The P1 cursor must be for a real table, not a pseudo-table. +#ifndef SQLITE_OMIT_CAST +/* Opcode: ToText P1 * * * * ** -** If P5 is positive and the jump is taken, then event counter -** number P5-1 in the prepared statement is incremented. +** Force the value in register P1 to be text. +** If the value is numeric, convert it to a string using the +** equivalent of printf(). Blob values are unchanged and +** are afterwards simply interpreted as text. ** -** See also: Prev +** A NULL value is not changed by this routine. It remains NULL. */ -/* Opcode: Prev P1 P2 * * P5 -** -** Back up cursor P1 so that it points to the previous key/data pair in its -** table or index. If there is no previous key/value pairs then fall through -** to the following instruction. But if the cursor backup was successful, -** jump immediately to P2. +case OP_ToText: { /* same as TK_TO_TEXT, in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + if( pIn1->flags & MEM_Null ) break; + assert( MEM_Str==(MEM_Blob>>3) ); + pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; + applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); + rc = ExpandBlob(pIn1); + assert( pIn1->flags & MEM_Str || db->mallocFailed ); + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + UPDATE_MAX_BLOBSIZE(pIn1); + break; +} + +/* Opcode: ToBlob P1 * * * * ** -** The P1 cursor must be for a real table, not a pseudo-table. +** Force the value in register P1 to be a BLOB. +** If the value is numeric, convert it to a string first. +** Strings are simply reinterpreted as blobs with no change +** to the underlying data. ** -** If P5 is positive and the jump is taken, then event counter -** number P5-1 in the prepared statement is incremented. +** A NULL value is not changed by this routine. It remains NULL. */ -case OP_Prev: /* jump */ -case OP_Next: { /* jump */ -#if 0 /* local variables moved into u.bm */ - VdbeCursor *pC; - BtCursor *pCrsr; - int res; -#endif /* local variables moved into u.bm */ - - CHECK_FOR_INTERRUPT; - assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p5<=ArraySize(p->aCounter) ); - u.bm.pC = p->apCsr[pOp->p1]; - if( u.bm.pC==0 ){ - break; /* See ticket #2273 */ - } - u.bm.pCrsr = u.bm.pC->pCursor; - if( u.bm.pCrsr==0 ){ - u.bm.pC->nullRow = 1; - break; - } - u.bm.res = 1; - assert( u.bm.pC->deferredMoveto==0 ); - rc = pOp->opcode==OP_Next ? sqlite3BtreeNext(u.bm.pCrsr, &u.bm.res) : - sqlite3BtreePrevious(u.bm.pCrsr, &u.bm.res); - u.bm.pC->nullRow = (u8)u.bm.res; - u.bm.pC->cacheStatus = CACHE_STALE; - if( u.bm.res==0 ){ - pc = pOp->p2 - 1; - if( pOp->p5 ) p->aCounter[pOp->p5-1]++; -#ifdef SQLITE_TEST - sqlite3_search_count++; -#endif +case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Null ) break; + if( (pIn1->flags & MEM_Blob)==0 ){ + applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); + assert( pIn1->flags & MEM_Str || db->mallocFailed ); + MemSetTypeFlag(pIn1, MEM_Blob); + }else{ + pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob); } - u.bm.pC->rowidIsValid = 0; + UPDATE_MAX_BLOBSIZE(pIn1); break; } -/* Opcode: IdxInsert P1 P2 P3 * P5 -** -** Register P2 holds a SQL index key made using the -** MakeRecord instructions. This opcode writes that key -** into the index P1. Data for the entry is nil. +/* Opcode: ToNumeric P1 * * * * ** -** P3 is a flag that provides a hint to the b-tree layer that this -** insert is likely to be an append. +** Force the value in register P1 to be numeric (either an +** integer or a floating-point number.) +** If the value is text or blob, try to convert it to an using the +** equivalent of atoi() or atof() and store 0 if no such conversion +** is possible. ** -** This instruction only works for indices. The equivalent instruction -** for tables is OP_Insert. +** A NULL value is not changed by this routine. It remains NULL. */ -case OP_IdxInsert: { /* in2 */ -#if 0 /* local variables moved into u.bn */ - VdbeCursor *pC; - BtCursor *pCrsr; - int nKey; - const char *zKey; -#endif /* local variables moved into u.bn */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bn.pC = p->apCsr[pOp->p1]; - assert( u.bn.pC!=0 ); - pIn2 = &aMem[pOp->p2]; - assert( pIn2->flags & MEM_Blob ); - u.bn.pCrsr = u.bn.pC->pCursor; - if( ALWAYS(u.bn.pCrsr!=0) ){ - assert( u.bn.pC->isTable==0 ); - rc = ExpandBlob(pIn2); - if( rc==SQLITE_OK ){ - u.bn.nKey = pIn2->n; - u.bn.zKey = pIn2->z; - rc = sqlite3BtreeInsert(u.bn.pCrsr, u.bn.zKey, u.bn.nKey, "", 0, 0, pOp->p3, - ((pOp->p5 & OPFLAG_USESEEKRESULT) ? u.bn.pC->seekResult : 0) - ); - assert( u.bn.pC->deferredMoveto==0 ); - u.bn.pC->cacheStatus = CACHE_STALE; - } - } +case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ + pIn1 = &aMem[pOp->p1]; + sqlite3VdbeMemNumerify(pIn1); break; } +#endif /* SQLITE_OMIT_CAST */ -/* Opcode: IdxDelete P1 P2 P3 * * +/* Opcode: ToInt P1 * * * * ** -** The content of P3 registers starting at register P2 form -** an unpacked index key. This opcode removes that entry from the -** index opened by cursor P1. +** Force the value in register P1 to be an integer. If +** The value is currently a real number, drop its fractional part. +** If the value is text or blob, try to convert it to an integer using the +** equivalent of atoi() and store 0 if no such conversion is possible. +** +** A NULL value is not changed by this routine. It remains NULL. */ -case OP_IdxDelete: { -#if 0 /* local variables moved into u.bo */ - VdbeCursor *pC; - BtCursor *pCrsr; - int res; - UnpackedRecord r; -#endif /* local variables moved into u.bo */ - - assert( pOp->p3>0 ); - assert( pOp->p2>0 && pOp->p2+pOp->p3<=p->nMem+1 ); - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bo.pC = p->apCsr[pOp->p1]; - assert( u.bo.pC!=0 ); - u.bo.pCrsr = u.bo.pC->pCursor; - if( ALWAYS(u.bo.pCrsr!=0) ){ - u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo; - u.bo.r.nField = (u16)pOp->p3; - u.bo.r.flags = 0; - u.bo.r.aMem = &aMem[pOp->p2]; -#ifdef SQLITE_DEBUG - { int i; for(i=0; ideferredMoveto==0 ); - u.bo.pC->cacheStatus = CACHE_STALE; +case OP_ToInt: { /* same as TK_TO_INT, in1 */ + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_Null)==0 ){ + sqlite3VdbeMemIntegerify(pIn1); } break; } -/* Opcode: IdxRowid P1 P2 * * * +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) +/* Opcode: ToReal P1 * * * * ** -** Write into register P2 an integer which is the last entry in the record at -** the end of the index key pointed to by cursor P1. This integer should be -** the rowid of the table entry to which this index entry points. +** Force the value in register P1 to be a floating point number. +** If The value is currently an integer, convert it. +** If the value is text or blob, try to convert it to an integer using the +** equivalent of atoi() and store 0.0 if no such conversion is possible. ** -** See also: Rowid, MakeRecord. +** A NULL value is not changed by this routine. It remains NULL. */ -case OP_IdxRowid: { /* out2-prerelease */ -#if 0 /* local variables moved into u.bp */ - BtCursor *pCrsr; - VdbeCursor *pC; - i64 rowid; -#endif /* local variables moved into u.bp */ - - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bp.pC = p->apCsr[pOp->p1]; - assert( u.bp.pC!=0 ); - u.bp.pCrsr = u.bp.pC->pCursor; - pOut->flags = MEM_Null; - if( ALWAYS(u.bp.pCrsr!=0) ){ - rc = sqlite3VdbeCursorMoveto(u.bp.pC); - if( NEVER(rc) ) goto abort_due_to_error; - assert( u.bp.pC->deferredMoveto==0 ); - assert( u.bp.pC->isTable==0 ); - if( !u.bp.pC->nullRow ){ - rc = sqlite3VdbeIdxRowid(db, u.bp.pCrsr, &u.bp.rowid); - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - pOut->u.i = u.bp.rowid; - pOut->flags = MEM_Int; - } +case OP_ToReal: { /* same as TK_TO_REAL, in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + if( (pIn1->flags & MEM_Null)==0 ){ + sqlite3VdbeMemRealify(pIn1); } break; } +#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ -/* Opcode: IdxGE P1 P2 P3 P4 P5 +/* Opcode: Lt P1 P2 P3 P4 P5 ** -** The P4 register values beginning with P3 form an unpacked index -** key that omits the ROWID. Compare this key value against the index -** that P1 is currently pointing to, ignoring the ROWID on the P1 index. +** Compare the values in register P1 and P3. If reg(P3)flags */ + u16 flags3; /* Copy of initial value of pIn3->flags */ +#endif /* local variables moved into u.ai */ - assert( pOp->p1>=0 && pOp->p1nCursor ); - u.bq.pC = p->apCsr[pOp->p1]; - assert( u.bq.pC!=0 ); - assert( u.bq.pC->isOrdered ); - if( ALWAYS(u.bq.pC->pCursor!=0) ){ - assert( u.bq.pC->deferredMoveto==0 ); - assert( pOp->p5==0 || pOp->p5==1 ); - assert( pOp->p4type==P4_INT32 ); - u.bq.r.pKeyInfo = u.bq.pC->pKeyInfo; - u.bq.r.nField = (u16)pOp->p4.i; - if( pOp->p5 ){ - u.bq.r.flags = UNPACKED_INCRKEY | UNPACKED_IGNORE_ROWID; - }else{ - u.bq.r.flags = UNPACKED_IGNORE_ROWID; - } - u.bq.r.aMem = &aMem[pOp->p3]; -#ifdef SQLITE_DEBUG - { int i; for(i=0; iopcode==OP_IdxLT ){ - u.bq.res = -u.bq.res; + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + u.ai.flags1 = pIn1->flags; + u.ai.flags3 = pIn3->flags; + if( (u.ai.flags1 | u.ai.flags3)&MEM_Null ){ + /* One or both operands are NULL */ + if( pOp->p5 & SQLITE_NULLEQ ){ + /* If SQLITE_NULLEQ is set (which will only happen if the operator is + ** OP_Eq or OP_Ne) then take the jump or not depending on whether + ** or not both operands are null. + */ + assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); + u.ai.res = (u.ai.flags1 & u.ai.flags3 & MEM_Null)==0; }else{ - assert( pOp->opcode==OP_IdxGE ); - u.bq.res++; + /* SQLITE_NULLEQ is clear and at least one operand is NULL, + ** then the result is always NULL. + ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. + */ + if( pOp->p5 & SQLITE_STOREP2 ){ + pOut = &aMem[pOp->p2]; + MemSetTypeFlag(pOut, MEM_Null); + REGISTER_TRACE(pOp->p2, pOut); + }else if( pOp->p5 & SQLITE_JUMPIFNULL ){ + pc = pOp->p2-1; + } + break; } - if( u.bq.res>0 ){ - pc = pOp->p2 - 1 ; + }else{ + /* Neither operand is NULL. Do a comparison. */ + u.ai.affinity = pOp->p5 & SQLITE_AFF_MASK; + if( u.ai.affinity ){ + applyAffinity(pIn1, u.ai.affinity, encoding); + applyAffinity(pIn3, u.ai.affinity, encoding); + if( db->mallocFailed ) goto no_mem; } + + assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); + ExpandBlob(pIn1); + ExpandBlob(pIn3); + u.ai.res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); + } + switch( pOp->opcode ){ + case OP_Eq: u.ai.res = u.ai.res==0; break; + case OP_Ne: u.ai.res = u.ai.res!=0; break; + case OP_Lt: u.ai.res = u.ai.res<0; break; + case OP_Le: u.ai.res = u.ai.res<=0; break; + case OP_Gt: u.ai.res = u.ai.res>0; break; + default: u.ai.res = u.ai.res>=0; break; + } + + if( pOp->p5 & SQLITE_STOREP2 ){ + pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); + MemSetTypeFlag(pOut, MEM_Int); + pOut->u.i = u.ai.res; + REGISTER_TRACE(pOp->p2, pOut); + }else if( u.ai.res ){ + pc = pOp->p2-1; } + + /* Undo any changes made by applyAffinity() to the input registers. */ + pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (u.ai.flags1&MEM_TypeMask); + pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (u.ai.flags3&MEM_TypeMask); break; } -/* Opcode: Destroy P1 P2 P3 * * -** -** Delete an entire database table or index whose root page in the database -** file is given by P1. -** -** The table being destroyed is in the main database file if P3==0. If -** P3==1 then the table to be clear is in the auxiliary database file -** that is used to store tables create using CREATE TEMPORARY TABLE. +/* Opcode: Permutation * * * P4 * ** -** If AUTOVACUUM is enabled then it is possible that another root page -** might be moved into the newly deleted root page in order to keep all -** root pages contiguous at the beginning of the database. The former -** value of the root page that moved - its value before the move occurred - -** is stored in register P2. If no page -** movement was required (because the table being dropped was already -** the last one in the database) then a zero is stored in register P2. -** If AUTOVACUUM is disabled then a zero is stored in register P2. +** Set the permutation used by the OP_Compare operator to be the array +** of integers in P4. ** -** See also: Clear +** The permutation is only valid until the next OP_Permutation, OP_Compare, +** OP_Halt, or OP_ResultRow. Typically the OP_Permutation should occur +** immediately prior to the OP_Compare. */ -case OP_Destroy: { /* out2-prerelease */ -#if 0 /* local variables moved into u.br */ - int iMoved; - int iCnt; - Vdbe *pVdbe; - int iDb; -#endif /* local variables moved into u.br */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - u.br.iCnt = 0; - for(u.br.pVdbe=db->pVdbe; u.br.pVdbe; u.br.pVdbe = u.br.pVdbe->pNext){ - if( u.br.pVdbe->magic==VDBE_MAGIC_RUN && u.br.pVdbe->inVtabMethod<2 && u.br.pVdbe->pc>=0 ){ - u.br.iCnt++; - } - } -#else - u.br.iCnt = db->activeVdbeCnt; -#endif - pOut->flags = MEM_Null; - if( u.br.iCnt>1 ){ - rc = SQLITE_LOCKED; - p->errorAction = OE_Abort; - }else{ - u.br.iDb = pOp->p3; - assert( u.br.iCnt==1 ); - assert( (p->btreeMask & (1<aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved); - pOut->flags = MEM_Int; - pOut->u.i = u.br.iMoved; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( rc==SQLITE_OK && u.br.iMoved!=0 ){ - sqlite3RootPageMoved(&db->aDb[u.br.iDb], u.br.iMoved, pOp->p1); - resetSchemaOnFault = 1; - } -#endif - } +case OP_Permutation: { + assert( pOp->p4type==P4_INTARRAY ); + assert( pOp->p4.ai ); + aPermute = pOp->p4.ai; break; } -/* Opcode: Clear P1 P2 P3 -** -** Delete all contents of the database table or index whose root page -** in the database file is given by P1. But, unlike Destroy, do not -** remove the table or index from the database file. +/* Opcode: Compare P1 P2 P3 P4 * ** -** The table being clear is in the main database file if P2==0. If -** P2==1 then the table to be clear is in the auxiliary database file -** that is used to store tables create using CREATE TEMPORARY TABLE. +** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this +** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of +** the comparison for use by the next OP_Jump instruct. ** -** If the P3 value is non-zero, then the table referred to must be an -** intkey table (an SQL table, not an index). In this case the row change -** count is incremented by the number of rows in the table being cleared. -** If P3 is greater than zero, then the value stored in register P3 is -** also incremented by the number of rows in the table being cleared. +** P4 is a KeyInfo structure that defines collating sequences and sort +** orders for the comparison. The permutation applies to registers +** only. The KeyInfo elements are used sequentially. ** -** See also: Destroy +** The comparison is a sort comparison, so NULLs compare equal, +** NULLs are less than numbers, numbers are less than strings, +** and strings are less than blobs. */ -case OP_Clear: { -#if 0 /* local variables moved into u.bs */ - int nChange; -#endif /* local variables moved into u.bs */ +case OP_Compare: { +#if 0 /* local variables moved into u.aj */ + int n; + int i; + int p1; + int p2; + const KeyInfo *pKeyInfo; + int idx; + CollSeq *pColl; /* Collating sequence to use on this term */ + int bRev; /* True for DESCENDING sort order */ +#endif /* local variables moved into u.aj */ - u.bs.nChange = 0; - assert( (p->btreeMask & (1<p2))!=0 ); - rc = sqlite3BtreeClearTable( - db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0) - ); - if( pOp->p3 ){ - p->nChange += u.bs.nChange; - if( pOp->p3>0 ){ - assert( memIsValid(&aMem[pOp->p3]) ); - memAboutToChange(p, &aMem[pOp->p3]); - aMem[pOp->p3].u.i += u.bs.nChange; + u.aj.n = pOp->p3; + u.aj.pKeyInfo = pOp->p4.pKeyInfo; + assert( u.aj.n>0 ); + assert( u.aj.pKeyInfo!=0 ); + u.aj.p1 = pOp->p1; + u.aj.p2 = pOp->p2; +#if SQLITE_DEBUG + if( aPermute ){ + int k, mx = 0; + for(k=0; kmx ) mx = aPermute[k]; + assert( u.aj.p1>0 && u.aj.p1+mx<=p->nMem+1 ); + assert( u.aj.p2>0 && u.aj.p2+mx<=p->nMem+1 ); + }else{ + assert( u.aj.p1>0 && u.aj.p1+u.aj.n<=p->nMem+1 ); + assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 ); + } +#endif /* SQLITE_DEBUG */ + for(u.aj.i=0; u.aj.inField ); + u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i]; + u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i]; + iCompare = sqlite3MemCompare(&aMem[u.aj.p1+u.aj.idx], &aMem[u.aj.p2+u.aj.idx], u.aj.pColl); + if( iCompare ){ + if( u.aj.bRev ) iCompare = -iCompare; + break; } } + aPermute = 0; break; } -/* Opcode: CreateTable P1 P2 * * * +/* Opcode: Jump P1 P2 P3 * * ** -** Allocate a new table in the main database file if P1==0 or in the -** auxiliary database file if P1==1 or in an attached database if -** P1>1. Write the root page number of the new table into -** register P2 +** Jump to the instruction at address P1, P2, or P3 depending on whether +** in the most recent OP_Compare instruction the P1 vector was less than +** equal to, or greater than the P2 vector, respectively. +*/ +case OP_Jump: { /* jump */ + if( iCompare<0 ){ + pc = pOp->p1 - 1; + }else if( iCompare==0 ){ + pc = pOp->p2 - 1; + }else{ + pc = pOp->p3 - 1; + } + break; +} + +/* Opcode: And P1 P2 P3 * * ** -** The difference between a table and an index is this: A table must -** have a 4-byte integer key and can have arbitrary data. An index -** has an arbitrary key but no data. +** Take the logical AND of the values in registers P1 and P2 and +** write the result into register P3. ** -** See also: CreateIndex +** If either P1 or P2 is 0 (false) then the result is 0 even if +** the other input is NULL. A NULL and true or two NULLs give +** a NULL output. */ -/* Opcode: CreateIndex P1 P2 * * * +/* Opcode: Or P1 P2 P3 * * ** -** Allocate a new index in the main database file if P1==0 or in the -** auxiliary database file if P1==1 or in an attached database if -** P1>1. Write the root page number of the new table into -** register P2. +** Take the logical OR of the values in register P1 and P2 and +** store the answer in register P3. ** -** See documentation on OP_CreateTable for additional information. +** If either P1 or P2 is nonzero (true) then the result is 1 (true) +** even if the other input is NULL. A NULL and false or two NULLs +** give a NULL output. */ -case OP_CreateIndex: /* out2-prerelease */ -case OP_CreateTable: { /* out2-prerelease */ -#if 0 /* local variables moved into u.bt */ - int pgno; - int flags; - Db *pDb; -#endif /* local variables moved into u.bt */ +case OP_And: /* same as TK_AND, in1, in2, out3 */ +case OP_Or: { /* same as TK_OR, in1, in2, out3 */ +#if 0 /* local variables moved into u.ak */ + int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ +#endif /* local variables moved into u.ak */ - u.bt.pgno = 0; - assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (1<p1))!=0 ); - u.bt.pDb = &db->aDb[pOp->p1]; - assert( u.bt.pDb->pBt!=0 ); - if( pOp->opcode==OP_CreateTable ){ - /* u.bt.flags = BTREE_INTKEY; */ - u.bt.flags = BTREE_INTKEY; + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Null ){ + u.ak.v1 = 2; }else{ - u.bt.flags = BTREE_BLOBKEY; + u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0; + } + pIn2 = &aMem[pOp->p2]; + if( pIn2->flags & MEM_Null ){ + u.ak.v2 = 2; + }else{ + u.ak.v2 = sqlite3VdbeIntValue(pIn2)!=0; + } + if( pOp->opcode==OP_And ){ + static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; + u.ak.v1 = and_logic[u.ak.v1*3+u.ak.v2]; + }else{ + static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 }; + u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2]; + } + pOut = &aMem[pOp->p3]; + if( u.ak.v1==2 ){ + MemSetTypeFlag(pOut, MEM_Null); + }else{ + pOut->u.i = u.ak.v1; + MemSetTypeFlag(pOut, MEM_Int); } - rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags); - pOut->u.i = u.bt.pgno; break; } -/* Opcode: ParseSchema P1 P2 * P4 * -** -** Read and parse all entries from the SQLITE_MASTER table of database P1 -** that match the WHERE clause P4. P2 is the "force" flag. Always do -** the parsing if P2 is true. If P2 is false, then this routine is a -** no-op if the schema is not currently loaded. In other words, if P2 -** is false, the SQLITE_MASTER table is only parsed if the rest of the -** schema is already loaded into the symbol table. +/* Opcode: Not P1 P2 * * * ** -** This opcode invokes the parser to create a new virtual machine, -** then runs the new virtual machine. It is thus a re-entrant opcode. +** Interpret the value in register P1 as a boolean value. Store the +** boolean complement in register P2. If the value in register P1 is +** NULL, then a NULL is stored in P2. */ -case OP_ParseSchema: { -#if 0 /* local variables moved into u.bu */ - int iDb; - const char *zMaster; - char *zSql; - InitData initData; -#endif /* local variables moved into u.bu */ - - u.bu.iDb = pOp->p1; - assert( u.bu.iDb>=0 && u.bu.iDbnDb ); - - /* If pOp->p2 is 0, then this opcode is being executed to read a - ** single row, for example the row corresponding to a new index - ** created by this VDBE, from the sqlite_master table. It only - ** does this if the corresponding in-memory schema is currently - ** loaded. Otherwise, the new index definition can be loaded along - ** with the rest of the schema when it is required. - ** - ** Although the mutex on the BtShared object that corresponds to - ** database u.bu.iDb (the database containing the sqlite_master table - ** read by this instruction) is currently held, it is necessary to - ** obtain the mutexes on all attached databases before checking if - ** the schema of u.bu.iDb is loaded. This is because, at the start of - ** the sqlite3_exec() call below, SQLite will invoke - ** sqlite3BtreeEnterAll(). If all mutexes are not already held, the - ** u.bu.iDb mutex may be temporarily released to avoid deadlock. If - ** this happens, then some other thread may delete the in-memory - ** schema of database u.bu.iDb before the SQL statement runs. The schema - ** will not be reloaded becuase the db->init.busy flag is set. This - ** can result in a "no such table: sqlite_master" or "malformed - ** database schema" error being returned to the user. - */ - assert( sqlite3BtreeHoldsMutex(db->aDb[u.bu.iDb].pBt) ); - sqlite3BtreeEnterAll(db); - if( pOp->p2 || DbHasProperty(db, u.bu.iDb, DB_SchemaLoaded) ){ - u.bu.zMaster = SCHEMA_TABLE(u.bu.iDb); - u.bu.initData.db = db; - u.bu.initData.iDb = pOp->p1; - u.bu.initData.pzErrMsg = &p->zErrMsg; - u.bu.zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", - db->aDb[u.bu.iDb].zName, u.bu.zMaster, pOp->p4.z); - if( u.bu.zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - assert( db->init.busy==0 ); - db->init.busy = 1; - u.bu.initData.rc = SQLITE_OK; - assert( !db->mallocFailed ); - rc = sqlite3_exec(db, u.bu.zSql, sqlite3InitCallback, &u.bu.initData, 0); - if( rc==SQLITE_OK ) rc = u.bu.initData.rc; - sqlite3DbFree(db, u.bu.zSql); - db->init.busy = 0; - } - } - sqlite3BtreeLeaveAll(db); - if( rc==SQLITE_NOMEM ){ - goto no_mem; +case OP_Not: { /* same as TK_NOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); } break; } -#if !defined(SQLITE_OMIT_ANALYZE) -/* Opcode: LoadAnalysis P1 * * * * +/* Opcode: BitNot P1 P2 * * * ** -** Read the sqlite_stat1 table for database P1 and load the content -** of that table into the internal index hash table. This will cause -** the analysis to be used when preparing all subsequent queries. +** Interpret the content of register P1 as an integer. Store the +** ones-complement of the P1 value into register P2. If P1 holds +** a NULL then store a NULL in P2. */ -case OP_LoadAnalysis: { - assert( pOp->p1>=0 && pOp->p1nDb ); - rc = sqlite3AnalysisLoad(db, pOp->p1); - break; +case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1)); + } + break; } -#endif /* !defined(SQLITE_OMIT_ANALYZE) */ -/* Opcode: DropTable P1 * * P4 * +/* Opcode: If P1 P2 P3 * * ** -** Remove the internal (in-memory) data structures that describe -** the table named P4 in database P1. This is called after a table -** is dropped in order to keep the internal representation of the -** schema consistent with what is on disk. +** Jump to P2 if the value in register P1 is true. The value +** is considered true if it is numeric and non-zero. If the value +** in P1 is NULL then take the jump if P3 is true. */ -case OP_DropTable: { - sqlite3UnlinkAndDeleteTable(db, pOp->p1, pOp->p4.z); +/* Opcode: IfNot P1 P2 P3 * * +** +** Jump to P2 if the value in register P1 is False. The value +** is considered true if it has a numeric value of zero. If the value +** in P1 is NULL then take the jump if P3 is true. +*/ +case OP_If: /* jump, in1 */ +case OP_IfNot: { /* jump, in1 */ +#if 0 /* local variables moved into u.al */ + int c; +#endif /* local variables moved into u.al */ + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & MEM_Null ){ + u.al.c = pOp->p3; + }else{ +#ifdef SQLITE_OMIT_FLOATING_POINT + u.al.c = sqlite3VdbeIntValue(pIn1)!=0; +#else + u.al.c = sqlite3VdbeRealValue(pIn1)!=0.0; +#endif + if( pOp->opcode==OP_IfNot ) u.al.c = !u.al.c; + } + if( u.al.c ){ + pc = pOp->p2-1; + } break; } -/* Opcode: DropIndex P1 * * P4 * +/* Opcode: IsNull P1 P2 * * * ** -** Remove the internal (in-memory) data structures that describe -** the index named P4 in database P1. This is called after an index -** is dropped in order to keep the internal representation of the -** schema consistent with what is on disk. +** Jump to P2 if the value in register P1 is NULL. */ -case OP_DropIndex: { - sqlite3UnlinkAndDeleteIndex(db, pOp->p1, pOp->p4.z); +case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_Null)!=0 ){ + pc = pOp->p2 - 1; + } break; } -/* Opcode: DropTrigger P1 * * P4 * +/* Opcode: NotNull P1 P2 * * * ** -** Remove the internal (in-memory) data structures that describe -** the trigger named P4 in database P1. This is called after a trigger -** is dropped in order to keep the internal representation of the -** schema consistent with what is on disk. +** Jump to P2 if the value in register P1 is not NULL. */ -case OP_DropTrigger: { - sqlite3UnlinkAndDeleteTrigger(db, pOp->p1, pOp->p4.z); +case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_Null)==0 ){ + pc = pOp->p2 - 1; + } break; } - -#ifndef SQLITE_OMIT_INTEGRITY_CHECK -/* Opcode: IntegrityCk P1 P2 P3 * P5 -** -** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +/* Opcode: Column P1 P2 P3 P4 P5 ** -** The register P3 contains the maximum number of allowed errors. -** At most reg(P3) errors will be reported. -** In other words, the analysis stops as soon as reg(P1) errors are -** seen. Reg(P1) is updated with the number of errors remaining. +** Interpret the data that cursor P1 points to as a structure built using +** the MakeRecord instruction. (See the MakeRecord opcode for additional +** information about the format of the data.) Extract the P2-th column +** from this record. If there are less that (P2+1) +** values in the record, extract a NULL. ** -** The root page numbers of all tables in the database are integer -** stored in reg(P1), reg(P1+1), reg(P1+2), .... There are P2 tables -** total. +** The value extracted is stored in register P3. ** -** If P5 is not zero, the check is done on the auxiliary database -** file, not the main database file. +** If the column contains fewer than P2 fields, then extract a NULL. Or, +** if the P4 argument is a P4_MEM use the value of the P4 argument as +** the result. ** -** This opcode is used to implement the integrity_check pragma. +** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor, +** then the cache of the cursor is reset prior to extracting the column. +** The first OP_Column against a pseudo-table after the value of the content +** register has changed should have this bit set. */ -case OP_IntegrityCk: { -#if 0 /* local variables moved into u.bv */ - int nRoot; /* Number of tables to check. (Number of root pages.) */ - int *aRoot; /* Array of rootpage numbers for tables to be checked */ - int j; /* Loop counter */ - int nErr; /* Number of errors reported */ - char *z; /* Text of the error report */ - Mem *pnErr; /* Register keeping track of errors remaining */ -#endif /* local variables moved into u.bv */ +case OP_Column: { +#if 0 /* local variables moved into u.am */ + u32 payloadSize; /* Number of bytes in the record */ + i64 payloadSize64; /* Number of bytes in the record */ + int p1; /* P1 value of the opcode */ + int p2; /* column number to retrieve */ + VdbeCursor *pC; /* The VDBE cursor */ + char *zRec; /* Pointer to complete record-data */ + BtCursor *pCrsr; /* The BTree cursor */ + u32 *aType; /* aType[i] holds the numeric type of the i-th column */ + u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ + int nField; /* number of fields in the record */ + int len; /* The length of the serialized data for the column */ + int i; /* Loop counter */ + char *zData; /* Part of the record being decoded */ + Mem *pDest; /* Where to write the extracted value */ + Mem sMem; /* For storing the record being decoded */ + u8 *zIdx; /* Index into header */ + u8 *zEndHdr; /* Pointer to first byte after the header */ + u32 offset; /* Offset into the data */ + u32 szField; /* Number of bytes in the content of a field */ + int szHdr; /* Size of the header size field at start of record */ + int avail; /* Number of bytes of available data */ + Mem *pReg; /* PseudoTable input register */ +#endif /* local variables moved into u.am */ - u.bv.nRoot = pOp->p2; - assert( u.bv.nRoot>0 ); - u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) ); - if( u.bv.aRoot==0 ) goto no_mem; + + u.am.p1 = pOp->p1; + u.am.p2 = pOp->p2; + u.am.pC = 0; + memset(&u.am.sMem, 0, sizeof(u.am.sMem)); + assert( u.am.p1nCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); - u.bv.pnErr = &aMem[pOp->p3]; - assert( (u.bv.pnErr->flags & MEM_Int)!=0 ); - assert( (u.bv.pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; - for(u.bv.j=0; u.bv.jp5nDb ); - assert( (p->btreeMask & (1<p5))!=0 ); - u.bv.z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, u.bv.aRoot, u.bv.nRoot, - (int)u.bv.pnErr->u.i, &u.bv.nErr); - sqlite3DbFree(db, u.bv.aRoot); - u.bv.pnErr->u.i -= u.bv.nErr; - sqlite3VdbeMemSetNull(pIn1); - if( u.bv.nErr==0 ){ - assert( u.bv.z==0 ); - }else if( u.bv.z==0 ){ - goto no_mem; + u.am.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.am.pDest); + MemSetTypeFlag(u.am.pDest, MEM_Null); + u.am.zRec = 0; + + /* This block sets the variable u.am.payloadSize to be the total number of + ** bytes in the record. + ** + ** u.am.zRec is set to be the complete text of the record if it is available. + ** The complete record text is always available for pseudo-tables + ** If the record is stored in a cursor, the complete record text + ** might be available in the u.am.pC->aRow cache. Or it might not be. + ** If the data is unavailable, u.am.zRec is set to NULL. + ** + ** We also compute the number of columns in the record. For cursors, + ** the number of columns is stored in the VdbeCursor.nField element. + */ + u.am.pC = p->apCsr[u.am.p1]; + assert( u.am.pC!=0 ); +#ifndef SQLITE_OMIT_VIRTUALTABLE + assert( u.am.pC->pVtabCursor==0 ); +#endif + u.am.pCrsr = u.am.pC->pCursor; + if( u.am.pCrsr!=0 ){ + /* The record is stored in a B-Tree */ + rc = sqlite3VdbeCursorMoveto(u.am.pC); + if( rc ) goto abort_due_to_error; + if( u.am.pC->nullRow ){ + u.am.payloadSize = 0; + }else if( u.am.pC->cacheStatus==p->cacheCtr ){ + u.am.payloadSize = u.am.pC->payloadSize; + u.am.zRec = (char*)u.am.pC->aRow; + }else if( u.am.pC->isIndex ){ + assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) ); + rc = sqlite3BtreeKeySize(u.am.pCrsr, &u.am.payloadSize64); + assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ + /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the + ** payload size, so it is impossible for u.am.payloadSize64 to be + ** larger than 32 bits. */ + assert( (u.am.payloadSize64 & SQLITE_MAX_U32)==(u64)u.am.payloadSize64 ); + u.am.payloadSize = (u32)u.am.payloadSize64; + }else{ + assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) ); + rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize); + assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + } + }else if( u.am.pC->pseudoTableReg>0 ){ + u.am.pReg = &aMem[u.am.pC->pseudoTableReg]; + assert( u.am.pReg->flags & MEM_Blob ); + assert( memIsValid(u.am.pReg) ); + u.am.payloadSize = u.am.pReg->n; + u.am.zRec = u.am.pReg->z; + u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr; + assert( u.am.payloadSize==0 || u.am.zRec!=0 ); }else{ - sqlite3VdbeMemSetStr(pIn1, u.bv.z, -1, SQLITE_UTF8, sqlite3_free); + /* Consider the row to be NULL */ + u.am.payloadSize = 0; } - UPDATE_MAX_BLOBSIZE(pIn1); - sqlite3VdbeChangeEncoding(pIn1, encoding); - break; -} -#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ -/* Opcode: RowSetAdd P1 P2 * * * -** -** Insert the integer value held by register P2 into a boolean index -** held in register P1. -** -** An assertion fails if P2 is not an integer. -*/ -case OP_RowSetAdd: { /* in1, in2 */ - pIn1 = &aMem[pOp->p1]; - pIn2 = &aMem[pOp->p2]; - assert( (pIn2->flags & MEM_Int)!=0 ); - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + /* If u.am.payloadSize is 0, then just store a NULL */ + if( u.am.payloadSize==0 ){ + assert( u.am.pDest->flags&MEM_Null ); + goto op_column_out; } - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); - break; -} - -/* Opcode: RowSetRead P1 P2 P3 * * -** -** Extract the smallest value from boolean index P1 and put that value into -** register P3. Or, if boolean index P1 is initially empty, leave P3 -** unchanged and jump to instruction P2. -*/ -case OP_RowSetRead: { /* jump, in1, out3 */ -#if 0 /* local variables moved into u.bw */ - i64 val; -#endif /* local variables moved into u.bw */ - CHECK_FOR_INTERRUPT; - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_RowSet)==0 - || sqlite3RowSetNext(pIn1->u.pRowSet, &u.bw.val)==0 - ){ - /* The boolean index is empty */ - sqlite3VdbeMemSetNull(pIn1); - pc = pOp->p2 - 1; - }else{ - /* A value was pulled from the index */ - sqlite3VdbeMemSetInt64(&aMem[pOp->p3], u.bw.val); + assert( db->aLimit[SQLITE_LIMIT_LENGTH]>=0 ); + if( u.am.payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; } - break; -} - -/* Opcode: RowSetTest P1 P2 P3 P4 -** -** Register P3 is assumed to hold a 64-bit integer value. If register P1 -** contains a RowSet object and that RowSet object contains -** the value held in P3, jump to register P2. Otherwise, insert the -** integer in P3 into the RowSet and continue on to the -** next opcode. -** -** The RowSet object is optimized for the case where successive sets -** of integers, where each set contains no duplicates. Each set -** of values is identified by a unique P4 value. The first set -** must have P4==0, the final set P4=-1. P4 must be either -1 or -** non-negative. For non-negative values of P4 only the lower 4 -** bits are significant. -** -** This allows optimizations: (a) when P4==0 there is no need to test -** the rowset object for P3, as it is guaranteed not to contain it, -** (b) when P4==-1 there is no need to insert the value, as it will -** never be tested for, and (c) when a value that is part of set X is -** inserted, there is no need to search to see if the same value was -** previously inserted as part of set X (only if it was previously -** inserted as part of some other set). -*/ -case OP_RowSetTest: { /* jump, in1, in3 */ -#if 0 /* local variables moved into u.bx */ - int iSet; - int exists; -#endif /* local variables moved into u.bx */ - pIn1 = &aMem[pOp->p1]; - pIn3 = &aMem[pOp->p3]; - u.bx.iSet = pOp->p4.i; - assert( pIn3->flags&MEM_Int ); + u.am.nField = u.am.pC->nField; + assert( u.am.p2flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; - } + u.am.aType = u.am.pC->aType; + if( u.am.pC->cacheStatus==p->cacheCtr ){ + u.am.aOffset = u.am.pC->aOffset; + }else{ + assert(u.am.aType); + u.am.avail = 0; + u.am.pC->aOffset = u.am.aOffset = &u.am.aType[u.am.nField]; + u.am.pC->payloadSize = u.am.payloadSize; + u.am.pC->cacheStatus = p->cacheCtr; - assert( pOp->p4type==P4_INT32 ); - assert( u.bx.iSet==-1 || u.bx.iSet>=0 ); - if( u.bx.iSet ){ - u.bx.exists = sqlite3RowSetTest(pIn1->u.pRowSet, - (u8)(u.bx.iSet>=0 ? u.bx.iSet & 0xf : 0xff), - pIn3->u.i); - if( u.bx.exists ){ - pc = pOp->p2 - 1; - break; + /* Figure out how many bytes are in the header */ + if( u.am.zRec ){ + u.am.zData = u.am.zRec; + }else{ + if( u.am.pC->isIndex ){ + u.am.zData = (char*)sqlite3BtreeKeyFetch(u.am.pCrsr, &u.am.avail); + }else{ + u.am.zData = (char*)sqlite3BtreeDataFetch(u.am.pCrsr, &u.am.avail); + } + /* If KeyFetch()/DataFetch() managed to get the entire payload, + ** save the payload in the u.am.pC->aRow cache. That will save us from + ** having to make additional calls to fetch the content portion of + ** the record. + */ + assert( u.am.avail>=0 ); + if( u.am.payloadSize <= (u32)u.am.avail ){ + u.am.zRec = u.am.zData; + u.am.pC->aRow = (u8*)u.am.zData; + }else{ + u.am.pC->aRow = 0; + } } - } - if( u.bx.iSet>=0 ){ - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); - } - break; -} - - -#ifndef SQLITE_OMIT_TRIGGER - -/* Opcode: Program P1 P2 P3 P4 * -** -** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). -** -** P1 contains the address of the memory cell that contains the first memory -** cell in an array of values used as arguments to the sub-program. P2 -** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address -** of a memory cell in this (the parent) VM that is used to allocate the -** memory required by the sub-vdbe at runtime. -** -** P4 is a pointer to the VM containing the trigger program. -*/ -case OP_Program: { /* jump */ -#if 0 /* local variables moved into u.by */ - int nMem; /* Number of memory registers for sub-program */ - int nByte; /* Bytes of runtime space required for sub-program */ - Mem *pRt; /* Register to allocate runtime space */ - Mem *pMem; /* Used to iterate through memory cells */ - Mem *pEnd; /* Last memory cell in new array */ - VdbeFrame *pFrame; /* New vdbe frame to execute in */ - SubProgram *pProgram; /* Sub-program to execute */ - void *t; /* Token identifying trigger */ -#endif /* local variables moved into u.by */ + /* The following assert is true in all cases accept when + ** the database file has been corrupted externally. + ** assert( u.am.zRec!=0 || u.am.avail>=u.am.payloadSize || u.am.avail>=9 ); */ + u.am.szHdr = getVarint32((u8*)u.am.zData, u.am.offset); - u.by.pProgram = pOp->p4.pProgram; - u.by.pRt = &aMem[pOp->p3]; - assert( memIsValid(u.by.pRt) ); - assert( u.by.pProgram->nOp>0 ); + /* Make sure a corrupt database has not given us an oversize header. + ** Do this now to avoid an oversize memory allocation. + ** + ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte + ** types use so much data space that there can only be 4096 and 32 of + ** them, respectively. So the maximum header length results from a + ** 3-byte type for each of the maximum of 32768 columns plus three + ** extra bytes for the header length itself. 32768*3 + 3 = 98307. + */ + if( u.am.offset > 98307 ){ + rc = SQLITE_CORRUPT_BKPT; + goto op_column_out; + } - /* If the p5 flag is clear, then recursive invocation of triggers is - ** disabled for backwards compatibility (p5 is set if this sub-program - ** is really a trigger, not a foreign key action, and the flag set - ** and cleared by the "PRAGMA recursive_triggers" command is clear). - ** - ** It is recursive invocation of triggers, at the SQL level, that is - ** disabled. In some cases a single trigger may generate more than one - ** SubProgram (if the trigger may be executed with more than one different - ** ON CONFLICT algorithm). SubProgram structures associated with a - ** single trigger all have the same value for the SubProgram.token - ** variable. */ - if( pOp->p5 ){ - u.by.t = u.by.pProgram->token; - for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent); - if( u.by.pFrame ) break; - } + /* Compute in u.am.len the number of bytes of data we need to read in order + ** to get u.am.nField type values. u.am.offset is an upper bound on this. But + ** u.am.nField might be significantly less than the true number of columns + ** in the table, and in that case, 5*u.am.nField+3 might be smaller than u.am.offset. + ** We want to minimize u.am.len in order to limit the size of the memory + ** allocation, especially if a corrupt database file has caused u.am.offset + ** to be oversized. Offset is limited to 98307 above. But 98307 might + ** still exceed Robson memory allocation limits on some configurations. + ** On systems that cannot tolerate large memory allocations, u.am.nField*5+3 + ** will likely be much smaller since u.am.nField will likely be less than + ** 20 or so. This insures that Robson memory allocation limits are + ** not exceeded even for corrupt database files. + */ + u.am.len = u.am.nField*5 + 3; + if( u.am.len > (int)u.am.offset ) u.am.len = (int)u.am.offset; - if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ - rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); - break; - } + /* The KeyFetch() or DataFetch() above are fast and will get the entire + ** record header in most cases. But they will fail to get the complete + ** record header if the record header does not fit on a single page + ** in the B-Tree. When that happens, use sqlite3VdbeMemFromBtree() to + ** acquire the complete header text. + */ + if( !u.am.zRec && u.am.availisIndex, &u.am.sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + u.am.zData = u.am.sMem.z; + } + u.am.zEndHdr = (u8 *)&u.am.zData[u.am.len]; + u.am.zIdx = (u8 *)&u.am.zData[u.am.szHdr]; - /* Register u.by.pRt is used to store the memory required to save the state - ** of the current program, and the memory required at runtime to execute - ** the trigger program. If this trigger has been fired before, then u.by.pRt - ** is already allocated. Otherwise, it must be initialized. */ - if( (u.by.pRt->flags&MEM_Frame)==0 ){ - /* SubProgram.nMem is set to the number of memory cells used by the - ** program stored in SubProgram.aOp. As well as these, one memory - ** cell is required for each cursor used by the program. Set local - ** variable u.by.nMem (and later, VdbeFrame.nChildMem) to this value. + /* Scan the header and use it to fill in the u.am.aType[] and u.am.aOffset[] + ** arrays. u.am.aType[u.am.i] will contain the type integer for the u.am.i-th + ** column and u.am.aOffset[u.am.i] will contain the u.am.offset from the beginning + ** of the record to the start of the data for the u.am.i-th column */ - u.by.nMem = u.by.pProgram->nMem + u.by.pProgram->nCsr; - u.by.nByte = ROUND8(sizeof(VdbeFrame)) - + u.by.nMem * sizeof(Mem) - + u.by.pProgram->nCsr * sizeof(VdbeCursor *); - u.by.pFrame = sqlite3DbMallocZero(db, u.by.nByte); - if( !u.by.pFrame ){ - goto no_mem; + for(u.am.i=0; u.am.iflags = MEM_Frame; - u.by.pRt->u.pFrame = u.by.pFrame; + sqlite3VdbeMemRelease(&u.am.sMem); + u.am.sMem.flags = MEM_Null; - u.by.pFrame->v = p; - u.by.pFrame->nChildMem = u.by.nMem; - u.by.pFrame->nChildCsr = u.by.pProgram->nCsr; - u.by.pFrame->pc = pc; - u.by.pFrame->aMem = p->aMem; - u.by.pFrame->nMem = p->nMem; - u.by.pFrame->apCsr = p->apCsr; - u.by.pFrame->nCursor = p->nCursor; - u.by.pFrame->aOp = p->aOp; - u.by.pFrame->nOp = p->nOp; - u.by.pFrame->token = u.by.pProgram->token; + /* If we have read more header data than was contained in the header, + ** or if the end of the last field appears to be past the end of the + ** record, or if the end of the last field appears to be before the end + ** of the record (when all fields present), then we must be dealing + ** with a corrupt database. + */ + if( (u.am.zIdx > u.am.zEndHdr) || (u.am.offset > u.am.payloadSize) + || (u.am.zIdx==u.am.zEndHdr && u.am.offset!=u.am.payloadSize) ){ + rc = SQLITE_CORRUPT_BKPT; + goto op_column_out; + } + } - u.by.pEnd = &VdbeFrameMem(u.by.pFrame)[u.by.pFrame->nChildMem]; - for(u.by.pMem=VdbeFrameMem(u.by.pFrame); u.by.pMem!=u.by.pEnd; u.by.pMem++){ - u.by.pMem->flags = MEM_Null; - u.by.pMem->db = db; + /* Get the column information. If u.am.aOffset[u.am.p2] is non-zero, then + ** deserialize the value from the record. If u.am.aOffset[u.am.p2] is zero, + ** then there are not enough fields in the record to satisfy the + ** request. In this case, set the value NULL or to P4 if P4 is + ** a pointer to a Mem object. + */ + if( u.am.aOffset[u.am.p2] ){ + assert( rc==SQLITE_OK ); + if( u.am.zRec ){ + sqlite3VdbeMemReleaseExternal(u.am.pDest); + sqlite3VdbeSerialGet((u8 *)&u.am.zRec[u.am.aOffset[u.am.p2]], u.am.aType[u.am.p2], u.am.pDest); + }else{ + u.am.len = sqlite3VdbeSerialTypeLen(u.am.aType[u.am.p2]); + sqlite3VdbeMemMove(&u.am.sMem, u.am.pDest); + rc = sqlite3VdbeMemFromBtree(u.am.pCrsr, u.am.aOffset[u.am.p2], u.am.len, u.am.pC->isIndex, &u.am.sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + u.am.zData = u.am.sMem.z; + sqlite3VdbeSerialGet((u8*)u.am.zData, u.am.aType[u.am.p2], u.am.pDest); } + u.am.pDest->enc = encoding; }else{ - u.by.pFrame = u.by.pRt->u.pFrame; - assert( u.by.pProgram->nMem+u.by.pProgram->nCsr==u.by.pFrame->nChildMem ); - assert( u.by.pProgram->nCsr==u.by.pFrame->nChildCsr ); - assert( pc==u.by.pFrame->pc ); + if( pOp->p4type==P4_MEM ){ + sqlite3VdbeMemShallowCopy(u.am.pDest, pOp->p4.pMem, MEM_Static); + }else{ + assert( u.am.pDest->flags&MEM_Null ); + } } - p->nFrame++; - u.by.pFrame->pParent = p->pFrame; - u.by.pFrame->lastRowid = db->lastRowid; - u.by.pFrame->nChange = p->nChange; - p->nChange = 0; - p->pFrame = u.by.pFrame; - p->aMem = aMem = &VdbeFrameMem(u.by.pFrame)[-1]; - p->nMem = u.by.pFrame->nChildMem; - p->nCursor = (u16)u.by.pFrame->nChildCsr; - p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; - p->aOp = aOp = u.by.pProgram->aOp; - p->nOp = u.by.pProgram->nOp; - pc = -1; + /* If we dynamically allocated space to hold the data (in the + ** sqlite3VdbeMemFromBtree() call above) then transfer control of that + ** dynamically allocated space over to the u.am.pDest structure. + ** This prevents a memory copy. + */ + if( u.am.sMem.zMalloc ){ + assert( u.am.sMem.z==u.am.sMem.zMalloc ); + assert( !(u.am.pDest->flags & MEM_Dyn) ); + assert( !(u.am.pDest->flags & (MEM_Blob|MEM_Str)) || u.am.pDest->z==u.am.sMem.z ); + u.am.pDest->flags &= ~(MEM_Ephem|MEM_Static); + u.am.pDest->flags |= MEM_Term; + u.am.pDest->z = u.am.sMem.z; + u.am.pDest->zMalloc = u.am.sMem.zMalloc; + } + + rc = sqlite3VdbeMemMakeWriteable(u.am.pDest); +op_column_out: + UPDATE_MAX_BLOBSIZE(u.am.pDest); + REGISTER_TRACE(pOp->p3, u.am.pDest); break; } -/* Opcode: Param P1 P2 * * * +/* Opcode: Affinity P1 P2 * P4 * ** -** This opcode is only ever present in sub-programs called via the -** OP_Program instruction. Copy a value currently stored in a memory -** cell of the calling (parent) frame to cell P2 in the current frames -** address space. This is used by trigger programs to access the new.* -** and old.* values. +** Apply affinities to a range of P2 registers starting with P1. ** -** The address of the cell in the parent frame is determined by adding -** the value of the P1 argument to the value of the P1 argument to the -** calling OP_Program instruction. +** P4 is a string that is P2 characters long. The nth character of the +** string indicates the column affinity that should be used for the nth +** memory cell in the range. */ -case OP_Param: { /* out2-prerelease */ -#if 0 /* local variables moved into u.bz */ - VdbeFrame *pFrame; - Mem *pIn; -#endif /* local variables moved into u.bz */ - u.bz.pFrame = p->pFrame; - u.bz.pIn = &u.bz.pFrame->aMem[pOp->p1 + u.bz.pFrame->aOp[u.bz.pFrame->pc].p1]; - sqlite3VdbeMemShallowCopy(pOut, u.bz.pIn, MEM_Ephem); - break; -} - -#endif /* #ifndef SQLITE_OMIT_TRIGGER */ +case OP_Affinity: { +#if 0 /* local variables moved into u.an */ + const char *zAffinity; /* The affinity to be applied */ + char cAff; /* A single character of affinity */ +#endif /* local variables moved into u.an */ -#ifndef SQLITE_OMIT_FOREIGN_KEY -/* Opcode: FkCounter P1 P2 * * * -** -** Increment a "constraint counter" by P2 (P2 may be negative or positive). -** If P1 is non-zero, the database constraint counter is incremented -** (deferred foreign key constraints). Otherwise, if P1 is zero, the -** statement counter is incremented (immediate foreign key constraints). -*/ -case OP_FkCounter: { - if( pOp->p1 ){ - db->nDeferredCons += pOp->p2; - }else{ - p->nFkConstraint += pOp->p2; + u.an.zAffinity = pOp->p4.z; + assert( u.an.zAffinity!=0 ); + assert( u.an.zAffinity[pOp->p2]==0 ); + pIn1 = &aMem[pOp->p1]; + while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){ + assert( pIn1 <= &p->aMem[p->nMem] ); + assert( memIsValid(pIn1) ); + ExpandBlob(pIn1); + applyAffinity(pIn1, u.an.cAff, encoding); + pIn1++; } break; } -/* Opcode: FkIfZero P1 P2 * * * +/* Opcode: MakeRecord P1 P2 P3 P4 * ** -** This opcode tests if a foreign key constraint-counter is currently zero. -** If so, jump to instruction P2. Otherwise, fall through to the next -** instruction. +** Convert P2 registers beginning with P1 into the [record format] +** use as a data record in a database table or as a key +** in an index. The OP_Column opcode can decode the record later. ** -** If P1 is non-zero, then the jump is taken if the database constraint-counter -** is zero (the one that counts deferred constraint violations). If P1 is -** zero, the jump is taken if the statement constraint-counter is zero -** (immediate foreign key constraint violations). -*/ -case OP_FkIfZero: { /* jump */ - if( pOp->p1 ){ - if( db->nDeferredCons==0 ) pc = pOp->p2-1; - }else{ - if( p->nFkConstraint==0 ) pc = pOp->p2-1; - } - break; -} -#endif /* #ifndef SQLITE_OMIT_FOREIGN_KEY */ - -#ifndef SQLITE_OMIT_AUTOINCREMENT -/* Opcode: MemMax P1 P2 * * * +** P4 may be a string that is P2 characters long. The nth character of the +** string indicates the column affinity that should be used for the nth +** field of the index key. ** -** P1 is a register in the root frame of this VM (the root frame is -** different from the current frame if this instruction is being executed -** within a sub-program). Set the value of register P1 to the maximum of -** its current value and the value in register P2. +** The mapping from character to affinity is given by the SQLITE_AFF_ +** macros defined in sqliteInt.h. ** -** This instruction throws an error if the memory cell is not initially -** an integer. +** If P4 is NULL then all index fields have the affinity NONE. */ -case OP_MemMax: { /* in2 */ -#if 0 /* local variables moved into u.ca */ - Mem *pIn1; - VdbeFrame *pFrame; -#endif /* local variables moved into u.ca */ - if( p->pFrame ){ - for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent); - u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1]; - }else{ - u.ca.pIn1 = &aMem[pOp->p1]; - } - assert( memIsValid(u.ca.pIn1) ); - sqlite3VdbeMemIntegerify(u.ca.pIn1); - pIn2 = &aMem[pOp->p2]; - sqlite3VdbeMemIntegerify(pIn2); - if( u.ca.pIn1->u.iu.i){ - u.ca.pIn1->u.i = pIn2->u.i; - } - break; -} -#endif /* SQLITE_OMIT_AUTOINCREMENT */ +case OP_MakeRecord: { +#if 0 /* local variables moved into u.ao */ + u8 *zNewRecord; /* A buffer to hold the data for the new record */ + Mem *pRec; /* The new record */ + u64 nData; /* Number of bytes of data space */ + int nHdr; /* Number of bytes of header space */ + i64 nByte; /* Data space required for this record */ + int nZero; /* Number of zero bytes at the end of the record */ + int nVarint; /* Number of bytes in a varint */ + u32 serial_type; /* Type field */ + Mem *pData0; /* First field to be combined into the record */ + Mem *pLast; /* Last field of the record */ + int nField; /* Number of fields in the record */ + char *zAffinity; /* The affinity string for the record */ + int file_format; /* File format to use for encoding */ + int i; /* Space used in zNewRecord[] */ + int len; /* Length of a field */ +#endif /* local variables moved into u.ao */ -/* Opcode: IfPos P1 P2 * * * -** -** If the value of register P1 is 1 or greater, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. -*/ -case OP_IfPos: { /* jump, in1 */ - pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags&MEM_Int ); - if( pIn1->u.i>0 ){ - pc = pOp->p2 - 1; - } - break; -} + /* Assuming the record contains N fields, the record format looks + ** like this: + ** + ** ------------------------------------------------------------------------ + ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | + ** ------------------------------------------------------------------------ + ** + ** Data(0) is taken from register P1. Data(1) comes from register P1+1 + ** and so froth. + ** + ** Each type field is a varint representing the serial type of the + ** corresponding data element (see sqlite3VdbeSerialType()). The + ** hdr-size field is also a varint which is the offset from the beginning + ** of the record to data0. + */ + u.ao.nData = 0; /* Number of bytes of data space */ + u.ao.nHdr = 0; /* Number of bytes of header space */ + u.ao.nZero = 0; /* Number of zero bytes at the end of the record */ + u.ao.nField = pOp->p1; + u.ao.zAffinity = pOp->p4.z; + assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 ); + u.ao.pData0 = &aMem[u.ao.nField]; + u.ao.nField = pOp->p2; + u.ao.pLast = &u.ao.pData0[u.ao.nField-1]; + u.ao.file_format = p->minWriteFileFormat; -/* Opcode: IfNeg P1 P2 * * * -** -** If the value of register P1 is less than zero, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. -*/ -case OP_IfNeg: { /* jump, in1 */ - pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags&MEM_Int ); - if( pIn1->u.i<0 ){ - pc = pOp->p2 - 1; + /* Identify the output register */ + assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); + + /* Loop through the elements that will make up the record to figure + ** out how much space is required for the new record. + */ + for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ + assert( memIsValid(u.ao.pRec) ); + if( u.ao.zAffinity ){ + applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding); + } + if( u.ao.pRec->flags&MEM_Zero && u.ao.pRec->n>0 ){ + sqlite3VdbeMemExpandBlob(u.ao.pRec); + } + u.ao.serial_type = sqlite3VdbeSerialType(u.ao.pRec, u.ao.file_format); + u.ao.len = sqlite3VdbeSerialTypeLen(u.ao.serial_type); + u.ao.nData += u.ao.len; + u.ao.nHdr += sqlite3VarintLen(u.ao.serial_type); + if( u.ao.pRec->flags & MEM_Zero ){ + /* Only pure zero-filled BLOBs can be input to this Opcode. + ** We do not allow blobs with a prefix and a zero-filled tail. */ + u.ao.nZero += u.ao.pRec->u.nZero; + }else if( u.ao.len ){ + u.ao.nZero = 0; + } } - break; -} -/* Opcode: IfZero P1 P2 P3 * * -** -** The register P1 must contain an integer. Add literal P3 to the -** value in register P1. If the result is exactly 0, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. -*/ -case OP_IfZero: { /* jump, in1 */ - pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; - if( pIn1->u.i==0 ){ - pc = pOp->p2 - 1; + /* Add the initial header varint and total the size */ + u.ao.nHdr += u.ao.nVarint = sqlite3VarintLen(u.ao.nHdr); + if( u.ao.nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; } - break; -} -/* Opcode: AggStep * P2 P3 P4 P5 -** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Use register -** P3 as the accumulator. -** -** The P5 arguments are taken from register P2 and its -** successors. -*/ -case OP_AggStep: { -#if 0 /* local variables moved into u.cb */ - int n; - int i; - Mem *pMem; - Mem *pRec; - sqlite3_context ctx; - sqlite3_value **apVal; -#endif /* local variables moved into u.cb */ + /* Make sure the output register has a buffer large enough to store + ** the new record. The output register (pOp->p3) is not allowed to + ** be one of the input registers (because the following call to + ** sqlite3VdbeMemGrow() could clobber the value before it is used). + */ + if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){ + goto no_mem; + } + u.ao.zNewRecord = (u8 *)pOut->z; - u.cb.n = pOp->p5; - assert( u.cb.n>=0 ); - u.cb.pRec = &aMem[pOp->p2]; - u.cb.apVal = p->apArg; - assert( u.cb.apVal || u.cb.n==0 ); - for(u.cb.i=0; u.cb.ip4.pFunc; - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3]; - u.cb.pMem->n++; - u.cb.ctx.s.flags = MEM_Null; - u.cb.ctx.s.z = 0; - u.cb.ctx.s.zMalloc = 0; - u.cb.ctx.s.xDel = 0; - u.cb.ctx.s.db = db; - u.cb.ctx.isError = 0; - u.cb.ctx.pColl = 0; - if( u.cb.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>p->aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - u.cb.ctx.pColl = pOp[-1].p4.pColl; + for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ /* serial data */ + u.ao.i += sqlite3VdbeSerialPut(&u.ao.zNewRecord[u.ao.i], (int)(u.ao.nByte-u.ao.i), u.ao.pRec,u.ao.file_format); } - (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); /* IMP: R-24505-23230 */ - if( u.cb.ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s)); - rc = u.cb.ctx.isError; + assert( u.ao.i==u.ao.nByte ); + + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + pOut->n = (int)u.ao.nByte; + pOut->flags = MEM_Blob | MEM_Dyn; + pOut->xDel = 0; + if( u.ao.nZero ){ + pOut->u.nZero = u.ao.nZero; + pOut->flags |= MEM_Zero; } - sqlite3VdbeMemRelease(&u.cb.ctx.s); + pOut->enc = SQLITE_UTF8; /* In case the blob is ever converted to text */ + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); break; } -/* Opcode: AggFinal P1 P2 * P4 * -** -** Execute the finalizer function for an aggregate. P1 is -** the memory location that is the accumulator for the aggregate. +/* Opcode: Count P1 P2 * * * ** -** P2 is the number of arguments that the step function takes and -** P4 is a pointer to the FuncDef for this function. The P2 -** argument is not used by this opcode. It is only there to disambiguate -** functions that can take varying numbers of arguments. The -** P4 argument is only needed for the degenerate case where -** the step function was not previously called. +** Store the number of entries (an integer value) in the table or index +** opened by cursor P1 in register P2 */ -case OP_AggFinal: { -#if 0 /* local variables moved into u.cc */ - Mem *pMem; -#endif /* local variables moved into u.cc */ - assert( pOp->p1>0 && pOp->p1<=p->nMem ); - u.cc.pMem = &aMem[pOp->p1]; - assert( (u.cc.pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); - rc = sqlite3VdbeMemFinalize(u.cc.pMem, pOp->p4.pFunc); - if( rc ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cc.pMem)); - } - sqlite3VdbeChangeEncoding(u.cc.pMem, encoding); - UPDATE_MAX_BLOBSIZE(u.cc.pMem); - if( sqlite3VdbeMemTooBig(u.cc.pMem) ){ - goto too_big; +#ifndef SQLITE_OMIT_BTREECOUNT +case OP_Count: { /* out2-prerelease */ +#if 0 /* local variables moved into u.ap */ + i64 nEntry; + BtCursor *pCrsr; +#endif /* local variables moved into u.ap */ + + u.ap.pCrsr = p->apCsr[pOp->p1]->pCursor; + if( u.ap.pCrsr ){ + rc = sqlite3BtreeCount(u.ap.pCrsr, &u.ap.nEntry); + }else{ + u.ap.nEntry = 0; } + pOut->u.i = u.ap.nEntry; break; } - -#ifndef SQLITE_OMIT_WAL -/* Opcode: Checkpoint P1 * * * * -** -** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. -*/ -case OP_Checkpoint: { - rc = sqlite3Checkpoint(db, pOp->p1); - break; -}; #endif -#ifndef SQLITE_OMIT_PRAGMA -/* Opcode: JournalMode P1 P2 P3 * P5 -** -** Change the journal mode of database P1 to P3. P3 must be one of the -** PAGER_JOURNALMODE_XXX values. If changing between the various rollback -** modes (delete, truncate, persist, off and memory), this is a simple -** operation. No IO is required. -** -** If changing into or out of WAL mode the procedure is more complicated. +/* Opcode: Savepoint P1 * * P4 * ** -** Write a string containing the final journal-mode to register P2. +** Open, release or rollback the savepoint named by parameter P4, depending +** on the value of P1. To open a new savepoint, P1==0. To release (commit) an +** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. */ -case OP_JournalMode: { /* out2-prerelease */ -#if 0 /* local variables moved into u.cd */ - Btree *pBt; /* Btree to change journal mode of */ - Pager *pPager; /* Pager associated with pBt */ - int eNew; /* New journal mode */ - int eOld; /* The old journal mode */ - const char *zFilename; /* Name of database file for pPager */ -#endif /* local variables moved into u.cd */ +case OP_Savepoint: { +#if 0 /* local variables moved into u.aq */ + int p1; /* Value of P1 operand */ + char *zName; /* Name of savepoint */ + int nName; + Savepoint *pNew; + Savepoint *pSavepoint; + Savepoint *pTmp; + int iSavepoint; + int ii; +#endif /* local variables moved into u.aq */ - u.cd.eNew = pOp->p3; - assert( u.cd.eNew==PAGER_JOURNALMODE_DELETE - || u.cd.eNew==PAGER_JOURNALMODE_TRUNCATE - || u.cd.eNew==PAGER_JOURNALMODE_PERSIST - || u.cd.eNew==PAGER_JOURNALMODE_OFF - || u.cd.eNew==PAGER_JOURNALMODE_MEMORY - || u.cd.eNew==PAGER_JOURNALMODE_WAL - || u.cd.eNew==PAGER_JOURNALMODE_QUERY - ); - assert( pOp->p1>=0 && pOp->p1nDb ); + u.aq.p1 = pOp->p1; + u.aq.zName = pOp->p4.z; + + /* Assert that the u.aq.p1 parameter is valid. Also that if there is no open + ** transaction, then there cannot be any savepoints. + */ + assert( db->pSavepoint==0 || db->autoCommit==0 ); + assert( u.aq.p1==SAVEPOINT_BEGIN||u.aq.p1==SAVEPOINT_RELEASE||u.aq.p1==SAVEPOINT_ROLLBACK ); + assert( db->pSavepoint || db->isTransactionSavepoint==0 ); + assert( checkSavepointCount(db) ); - /* This opcode is used in two places: PRAGMA journal_mode and ATTACH. - ** In PRAGMA journal_mode, the sqlite3VdbeUsesBtree() routine is called - ** when the statment is prepared and so p->aMutex.nMutex>0. All mutexes - ** are already acquired. But when used in ATTACH, sqlite3VdbeUsesBtree() - ** is not called when the statement is prepared because it requires the - ** iDb index of the database as a parameter, and the database has not - ** yet been attached so that index is unavailable. We have to wait - ** until runtime (now) to get the mutex on the newly attached database. - ** No other mutexes are required by the ATTACH command so this is safe - ** to do. - */ - assert( (p->btreeMask & (1<p1))!=0 || p->aMutex.nMutex==0 ); - if( p->aMutex.nMutex==0 ){ - /* This occurs right after ATTACH. Get a mutex on the newly ATTACHed - ** database. */ - sqlite3VdbeUsesBtree(p, pOp->p1); - sqlite3VdbeMutexArrayEnter(p); - } - - u.cd.pBt = db->aDb[pOp->p1].pBt; - u.cd.pPager = sqlite3BtreePager(u.cd.pBt); - u.cd.eOld = sqlite3PagerGetJournalMode(u.cd.pPager); - if( u.cd.eNew==PAGER_JOURNALMODE_QUERY ) u.cd.eNew = u.cd.eOld; - if( !sqlite3PagerOkToChangeJournalMode(u.cd.pPager) ) u.cd.eNew = u.cd.eOld; + if( u.aq.p1==SAVEPOINT_BEGIN ){ + if( db->writeVdbeCnt>0 ){ + /* A new savepoint cannot be created if there are active write + ** statements (i.e. open read/write incremental blob handles). + */ + sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else{ + u.aq.nName = sqlite3Strlen30(u.aq.zName); -#ifndef SQLITE_OMIT_WAL - u.cd.zFilename = sqlite3PagerFilename(u.cd.pPager); +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* This call is Ok even if this savepoint is actually a transaction + ** savepoint (and therefore should not prompt xSavepoint()) callbacks. + ** If this is a transaction savepoint being opened, it is guaranteed + ** that the db->aVTrans[] array is empty. */ + assert( db->autoCommit==0 || db->nVTrans==0 ); + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, + db->nStatement+db->nSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif - /* Do not allow a transition to journal_mode=WAL for a database - ** in temporary storage or if the VFS does not support shared memory - */ - if( u.cd.eNew==PAGER_JOURNALMODE_WAL - && (u.cd.zFilename[0]==0 /* Temp file */ - || !sqlite3PagerWalSupported(u.cd.pPager)) /* No shared-memory support */ - ){ - u.cd.eNew = u.cd.eOld; - } + /* Create a new savepoint structure. */ + u.aq.pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+u.aq.nName+1); + if( u.aq.pNew ){ + u.aq.pNew->zName = (char *)&u.aq.pNew[1]; + memcpy(u.aq.pNew->zName, u.aq.zName, u.aq.nName+1); - if( (u.cd.eNew!=u.cd.eOld) - && (u.cd.eOld==PAGER_JOURNALMODE_WAL || u.cd.eNew==PAGER_JOURNALMODE_WAL) - ){ - if( !db->autoCommit || db->activeVdbeCnt>1 ){ + /* If there is no open transaction, then mark this as a special + ** "transaction savepoint". */ + if( db->autoCommit ){ + db->autoCommit = 0; + db->isTransactionSavepoint = 1; + }else{ + db->nSavepoint++; + } + + /* Link the new savepoint into the database handle's list. */ + u.aq.pNew->pNext = db->pSavepoint; + db->pSavepoint = u.aq.pNew; + u.aq.pNew->nDeferredCons = db->nDeferredCons; + } + } + }else{ + u.aq.iSavepoint = 0; + + /* Find the named savepoint. If there is no such savepoint, then an + ** an error is returned to the user. */ + for( + u.aq.pSavepoint = db->pSavepoint; + u.aq.pSavepoint && sqlite3StrICmp(u.aq.pSavepoint->zName, u.aq.zName); + u.aq.pSavepoint = u.aq.pSavepoint->pNext + ){ + u.aq.iSavepoint++; + } + if( !u.aq.pSavepoint ){ + sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", u.aq.zName); rc = SQLITE_ERROR; + }else if( + db->writeVdbeCnt>0 || (u.aq.p1==SAVEPOINT_ROLLBACK && db->activeVdbeCnt>1) + ){ + /* It is not possible to release (commit) a savepoint if there are + ** active write statements. It is not possible to rollback a savepoint + ** if there are any active statements at all. + */ sqlite3SetString(&p->zErrMsg, db, - "cannot change %s wal mode from within a transaction", - (u.cd.eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") + "cannot %s savepoint - SQL statements in progress", + (u.aq.p1==SAVEPOINT_ROLLBACK ? "rollback": "release") ); - break; + rc = SQLITE_BUSY; }else{ - if( u.cd.eOld==PAGER_JOURNALMODE_WAL ){ - /* If leaving WAL mode, close the log file. If successful, the call - ** to PagerCloseWal() checkpoints and deletes the write-ahead-log - ** file. An EXCLUSIVE lock may still be held on the database file - ** after a successful return. - */ - rc = sqlite3PagerCloseWal(u.cd.pPager); - if( rc==SQLITE_OK ){ - sqlite3PagerSetJournalMode(u.cd.pPager, u.cd.eNew); + /* Determine whether or not this is a transaction savepoint. If so, + ** and this is a RELEASE command, then the current transaction + ** is committed. + */ + int isTransaction = u.aq.pSavepoint->pNext==0 && db->isTransactionSavepoint; + if( isTransaction && u.aq.p1==SAVEPOINT_RELEASE ){ + if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + goto vdbe_return; + } + db->autoCommit = 1; + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = pc; + db->autoCommit = 0; + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + db->isTransactionSavepoint = 0; + rc = p->rc; + }else{ + u.aq.iSavepoint = db->nSavepoint - u.aq.iSavepoint - 1; + for(u.aq.ii=0; u.aq.iinDb; u.aq.ii++){ + rc = sqlite3BtreeSavepoint(db->aDb[u.aq.ii].pBt, u.aq.p1, u.aq.iSavepoint); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + } + if( u.aq.p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetInternalSchema(db, -1); + db->flags = (db->flags | SQLITE_InternChanges); } - }else if( u.cd.eOld==PAGER_JOURNALMODE_MEMORY ){ - /* Cannot transition directly from MEMORY to WAL. Use mode OFF - ** as an intermediate */ - sqlite3PagerSetJournalMode(u.cd.pPager, PAGER_JOURNALMODE_OFF); } - /* Open a transaction on the database file. Regardless of the journal - ** mode, this transaction always uses a rollback journal. - */ - assert( sqlite3BtreeIsInTrans(u.cd.pBt)==0 ); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeSetVersion(u.cd.pBt, (u.cd.eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); + /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all + ** savepoints nested inside of the savepoint being operated on. */ + while( db->pSavepoint!=u.aq.pSavepoint ){ + u.aq.pTmp = db->pSavepoint; + db->pSavepoint = u.aq.pTmp->pNext; + sqlite3DbFree(db, u.aq.pTmp); + db->nSavepoint--; } - } - } -#endif /* ifndef SQLITE_OMIT_WAL */ - - if( rc ){ - u.cd.eNew = u.cd.eOld; - } - u.cd.eNew = sqlite3PagerSetJournalMode(u.cd.pPager, u.cd.eNew); - pOut = &aMem[pOp->p2]; - pOut->flags = MEM_Str|MEM_Static|MEM_Term; - pOut->z = (char *)sqlite3JournalModename(u.cd.eNew); - pOut->n = sqlite3Strlen30(pOut->z); - pOut->enc = SQLITE_UTF8; - sqlite3VdbeChangeEncoding(pOut, encoding); - break; -}; -#endif /* SQLITE_OMIT_PRAGMA */ - -#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum * * * * * -** -** Vacuum the entire database. This opcode will cause other virtual -** machines to be created and run. It may not be called from within -** a transaction. -*/ -case OP_Vacuum: { - rc = sqlite3RunVacuum(&p->zErrMsg, db); - break; -} -#endif - -#if !defined(SQLITE_OMIT_AUTOVACUUM) -/* Opcode: IncrVacuum P1 P2 * * * -** -** Perform a single step of the incremental vacuum procedure on -** the P1 database. If the vacuum has finished, jump to instruction -** P2. Otherwise, fall through to the next instruction. -*/ -case OP_IncrVacuum: { /* jump */ -#if 0 /* local variables moved into u.ce */ - Btree *pBt; -#endif /* local variables moved into u.ce */ - - assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (1<p1))!=0 ); - u.ce.pBt = db->aDb[pOp->p1].pBt; - rc = sqlite3BtreeIncrVacuum(u.ce.pBt); - if( rc==SQLITE_DONE ){ - pc = pOp->p2 - 1; - rc = SQLITE_OK; - } - break; -} -#endif - -/* Opcode: Expire P1 * * * * -** -** Cause precompiled statements to become expired. An expired statement -** fails with an error code of SQLITE_SCHEMA if it is ever executed -** (via sqlite3_step()). -** -** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, -** then only the currently executing statement is affected. -*/ -case OP_Expire: { - if( !pOp->p1 ){ - sqlite3ExpirePreparedStatements(db); - }else{ - p->expired = 1; - } - break; -} + /* If it is a RELEASE, then destroy the savepoint being operated on + ** too. If it is a ROLLBACK TO, then set the number of deferred + ** constraint violations present in the database to the value stored + ** when the savepoint was created. */ + if( u.aq.p1==SAVEPOINT_RELEASE ){ + assert( u.aq.pSavepoint==db->pSavepoint ); + db->pSavepoint = u.aq.pSavepoint->pNext; + sqlite3DbFree(db, u.aq.pSavepoint); + if( !isTransaction ){ + db->nSavepoint--; + } + }else{ + db->nDeferredCons = u.aq.pSavepoint->nDeferredCons; + } -#ifndef SQLITE_OMIT_SHARED_CACHE -/* Opcode: TableLock P1 P2 P3 P4 * -** -** Obtain a lock on a particular table. This instruction is only used when -** the shared-cache feature is enabled. -** -** P1 is the index of the database in sqlite3.aDb[] of the database -** on which the lock is acquired. A readlock is obtained if P3==0 or -** a write lock if P3==1. -** -** P2 contains the root-page of the table to lock. -** -** P4 contains a pointer to the name of the table being locked. This is only -** used to generate an error message if the lock cannot be obtained. -*/ -case OP_TableLock: { - u8 isWriteLock = (u8)pOp->p3; - if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ - int p1 = pOp->p1; - assert( p1>=0 && p1nDb ); - assert( (p->btreeMask & (1<aDb[p1].pBt, pOp->p2, isWriteLock); - if( (rc&0xFF)==SQLITE_LOCKED ){ - const char *z = pOp->p4.z; - sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); + if( !isTransaction ){ + rc = sqlite3VtabSavepoint(db, u.aq.p1, u.aq.iSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } } } - break; -} -#endif /* SQLITE_OMIT_SHARED_CACHE */ - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VBegin * * * P4 * -** -** P4 may be a pointer to an sqlite3_vtab structure. If so, call the -** xBegin method for that table. -** -** Also, whether or not P4 is set, check that this is not being called from -** within a callback to a virtual table xSync() method. If it is, the error -** code will be set to SQLITE_LOCKED. -*/ -case OP_VBegin: { -#if 0 /* local variables moved into u.cf */ - VTable *pVTab; -#endif /* local variables moved into u.cf */ - u.cf.pVTab = pOp->p4.pVtab; - rc = sqlite3VtabBegin(db, u.cf.pVTab); - if( u.cf.pVTab ) importVtabErrMsg(p, u.cf.pVTab->pVtab); - break; -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VCreate P1 * * P4 * -** -** P4 is the name of a virtual table in database P1. Call the xCreate method -** for that table. -*/ -case OP_VCreate: { - rc = sqlite3VtabCallCreate(db, pOp->p1, pOp->p4.z, &p->zErrMsg); break; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VDestroy P1 * * P4 * +/* Opcode: AutoCommit P1 P2 * * * ** -** P4 is the name of a virtual table in database P1. Call the xDestroy method -** of that table. -*/ -case OP_VDestroy: { - p->inVtabMethod = 2; - rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); - p->inVtabMethod = 0; - break; -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VOpen P1 * * P4 * +** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll +** back any currently active btree transactions. If there are any active +** VMs (apart from this one), then a ROLLBACK fails. A COMMIT fails if +** there are active writing VMs or active VMs that use shared cache. ** -** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. -** P1 is a cursor number. This opcode opens a cursor to the virtual -** table and stores that cursor in P1. +** This instruction causes the VM to halt. */ -case OP_VOpen: { -#if 0 /* local variables moved into u.cg */ - VdbeCursor *pCur; - sqlite3_vtab_cursor *pVtabCursor; - sqlite3_vtab *pVtab; - sqlite3_module *pModule; -#endif /* local variables moved into u.cg */ +case OP_AutoCommit: { +#if 0 /* local variables moved into u.ar */ + int desiredAutoCommit; + int iRollback; + int turnOnAC; +#endif /* local variables moved into u.ar */ - u.cg.pCur = 0; - u.cg.pVtabCursor = 0; - u.cg.pVtab = pOp->p4.pVtab->pVtab; - u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule; - assert(u.cg.pVtab && u.cg.pModule); - rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor); - importVtabErrMsg(p, u.cg.pVtab); - if( SQLITE_OK==rc ){ - /* Initialize sqlite3_vtab_cursor base class */ - u.cg.pVtabCursor->pVtab = u.cg.pVtab; + u.ar.desiredAutoCommit = pOp->p1; + u.ar.iRollback = pOp->p2; + u.ar.turnOnAC = u.ar.desiredAutoCommit && !db->autoCommit; + assert( u.ar.desiredAutoCommit==1 || u.ar.desiredAutoCommit==0 ); + assert( u.ar.desiredAutoCommit==1 || u.ar.iRollback==0 ); + assert( db->activeVdbeCnt>0 ); /* At least this one VM is active */ - /* Initialise vdbe cursor object */ - u.cg.pCur = allocateCursor(p, pOp->p1, 0, -1, 0); - if( u.cg.pCur ){ - u.cg.pCur->pVtabCursor = u.cg.pVtabCursor; - u.cg.pCur->pModule = u.cg.pVtabCursor->pVtab->pModule; + if( u.ar.turnOnAC && u.ar.iRollback && db->activeVdbeCnt>1 ){ + /* If this instruction implements a ROLLBACK and other VMs are + ** still running, and a transaction is active, return an error indicating + ** that the other VMs must complete first. + */ + sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else if( u.ar.turnOnAC && !u.ar.iRollback && db->writeVdbeCnt>0 ){ + /* If this instruction implements a COMMIT and other VMs are writing + ** return an error indicating that the other VMs must complete first. + */ + sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else if( u.ar.desiredAutoCommit!=db->autoCommit ){ + if( u.ar.iRollback ){ + assert( u.ar.desiredAutoCommit==1 ); + sqlite3RollbackAll(db); + db->autoCommit = 1; + }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + goto vdbe_return; }else{ - db->mallocFailed = 1; - u.cg.pModule->xClose(u.cg.pVtabCursor); + db->autoCommit = (u8)u.ar.desiredAutoCommit; + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = pc; + db->autoCommit = (u8)(1-u.ar.desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + } + assert( db->nStatement==0 ); + sqlite3CloseSavepoints(db); + if( p->rc==SQLITE_OK ){ + rc = SQLITE_DONE; + }else{ + rc = SQLITE_ERROR; } + goto vdbe_return; + }else{ + sqlite3SetString(&p->zErrMsg, db, + (!u.ar.desiredAutoCommit)?"cannot start a transaction within a transaction":( + (u.ar.iRollback)?"cannot rollback - no transaction is active": + "cannot commit - no transaction is active")); + + rc = SQLITE_ERROR; } break; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VFilter P1 P2 P3 P4 * +/* Opcode: Transaction P1 P2 * * * ** -** P1 is a cursor opened using VOpen. P2 is an address to jump to if -** the filtered result set is empty. +** Begin a transaction. The transaction ends when a Commit or Rollback +** opcode is encountered. Depending on the ON CONFLICT setting, the +** transaction might also be rolled back if an error is encountered. ** -** P4 is either NULL or a string that was generated by the xBestIndex -** method of the module. The interpretation of the P4 string is left -** to the module implementation. +** P1 is the index of the database file on which the transaction is +** started. Index 0 is the main database file and index 1 is the +** file used for temporary tables. Indices of 2 or more are used for +** attached databases. ** -** This opcode invokes the xFilter method on the virtual table specified -** by P1. The integer query plan parameter to xFilter is stored in register -** P3. Register P3+1 stores the argc parameter to be passed to the -** xFilter method. Registers P3+2..P3+1+argc are the argc -** additional parameters which are passed to -** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter. +** If P2 is non-zero, then a write-transaction is started. A RESERVED lock is +** obtained on the database file when a write-transaction is started. No +** other process can start another write transaction while this transaction is +** underway. Starting a write transaction also creates a rollback journal. A +** write transaction must be started before any changes can be made to the +** database. If P2 is 2 or greater then an EXCLUSIVE lock is also obtained +** on the file. ** -** A jump is made to P2 if the result set after filtering would be empty. +** If a write-transaction is started and the Vdbe.usesStmtJournal flag is +** true (this flag is set if the Vdbe may modify more than one row and may +** throw an ABORT exception), a statement transaction may also be opened. +** More specifically, a statement transaction is opened iff the database +** connection is currently not in autocommit mode, or if there are other +** active statements. A statement transaction allows the affects of this +** VDBE to be rolled back after an error without having to roll back the +** entire transaction. If no error is encountered, the statement transaction +** will automatically commit when the VDBE halts. +** +** If P2 is zero, then a read-lock is obtained on the database file. */ -case OP_VFilter: { /* jump */ -#if 0 /* local variables moved into u.ch */ - int nArg; - int iQuery; - const sqlite3_module *pModule; - Mem *pQuery; - Mem *pArgc; - sqlite3_vtab_cursor *pVtabCursor; - sqlite3_vtab *pVtab; - VdbeCursor *pCur; - int res; - int i; - Mem **apArg; -#endif /* local variables moved into u.ch */ - - u.ch.pQuery = &aMem[pOp->p3]; - u.ch.pArgc = &u.ch.pQuery[1]; - u.ch.pCur = p->apCsr[pOp->p1]; - assert( memIsValid(u.ch.pQuery) ); - REGISTER_TRACE(pOp->p3, u.ch.pQuery); - assert( u.ch.pCur->pVtabCursor ); - u.ch.pVtabCursor = u.ch.pCur->pVtabCursor; - u.ch.pVtab = u.ch.pVtabCursor->pVtab; - u.ch.pModule = u.ch.pVtab->pModule; +case OP_Transaction: { +#if 0 /* local variables moved into u.as */ + Btree *pBt; +#endif /* local variables moved into u.as */ - /* Grab the index number and argc parameters */ - assert( (u.ch.pQuery->flags&MEM_Int)!=0 && u.ch.pArgc->flags==MEM_Int ); - u.ch.nArg = (int)u.ch.pArgc->u.i; - u.ch.iQuery = (int)u.ch.pQuery->u.i; + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + u.as.pBt = db->aDb[pOp->p1].pBt; - /* Invoke the xFilter method */ - { - u.ch.res = 0; - u.ch.apArg = p->apArg; - for(u.ch.i = 0; u.ch.ip2); + if( rc==SQLITE_BUSY ){ + p->pc = pc; + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; } - - p->inVtabMethod = 1; - rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg); - p->inVtabMethod = 0; - importVtabErrMsg(p, u.ch.pVtab); - if( rc==SQLITE_OK ){ - u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; } - if( u.ch.res ){ - pc = pOp->p2 - 1; + if( pOp->p2 && p->usesStmtJournal + && (db->autoCommit==0 || db->activeVdbeCnt>1) + ){ + assert( sqlite3BtreeIsInTrans(u.as.pBt) ); + if( p->iStatement==0 ){ + assert( db->nStatement>=0 && db->nSavepoint>=0 ); + db->nStatement++; + p->iStatement = db->nSavepoint + db->nStatement; + } + + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement-1); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginStmt(u.as.pBt, p->iStatement); + } + + /* Store the current value of the database handles deferred constraint + ** counter. If the statement transaction needs to be rolled back, + ** the value of this counter needs to be restored too. */ + p->nStmtDefCons = db->nDeferredCons; } } - u.ch.pCur->nullRow = 0; - break; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VColumn P1 P2 P3 * * +/* Opcode: ReadCookie P1 P2 P3 * * ** -** Store the value of the P2-th column of -** the row of the virtual-table that the -** P1 cursor is pointing to into register P3. +** Read cookie number P3 from database P1 and write it into register P2. +** P3==1 is the schema version. P3==2 is the database format. +** P3==3 is the recommended pager cache size, and so forth. P1==0 is +** the main database file and P1==1 is the database file used to store +** temporary tables. +** +** There must be a read-lock on the database (either a transaction +** must be started or there must be an open cursor) before +** executing this instruction. */ -case OP_VColumn: { -#if 0 /* local variables moved into u.ci */ - sqlite3_vtab *pVtab; - const sqlite3_module *pModule; - Mem *pDest; - sqlite3_context sContext; -#endif /* local variables moved into u.ci */ - - VdbeCursor *pCur = p->apCsr[pOp->p1]; - assert( pCur->pVtabCursor ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - u.ci.pDest = &aMem[pOp->p3]; - memAboutToChange(p, u.ci.pDest); - if( pCur->nullRow ){ - sqlite3VdbeMemSetNull(u.ci.pDest); - break; - } - u.ci.pVtab = pCur->pVtabCursor->pVtab; - u.ci.pModule = u.ci.pVtab->pModule; - assert( u.ci.pModule->xColumn ); - memset(&u.ci.sContext, 0, sizeof(u.ci.sContext)); - - /* The output cell may already have a buffer allocated. Move - ** the current contents to u.ci.sContext.s so in case the user-function - ** can use the already allocated buffer instead of allocating a - ** new one. - */ - sqlite3VdbeMemMove(&u.ci.sContext.s, u.ci.pDest); - MemSetTypeFlag(&u.ci.sContext.s, MEM_Null); - - rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2); - importVtabErrMsg(p, u.ci.pVtab); - if( u.ci.sContext.isError ){ - rc = u.ci.sContext.isError; - } +case OP_ReadCookie: { /* out2-prerelease */ +#if 0 /* local variables moved into u.at */ + int iMeta; + int iDb; + int iCookie; +#endif /* local variables moved into u.at */ - /* Copy the result of the function to the P3 register. We - ** do this regardless of whether or not an error occurred to ensure any - ** dynamic allocation in u.ci.sContext.s (a Mem struct) is released. - */ - sqlite3VdbeChangeEncoding(&u.ci.sContext.s, encoding); - sqlite3VdbeMemMove(u.ci.pDest, &u.ci.sContext.s); - REGISTER_TRACE(pOp->p3, u.ci.pDest); - UPDATE_MAX_BLOBSIZE(u.ci.pDest); + u.at.iDb = pOp->p1; + u.at.iCookie = pOp->p3; + assert( pOp->p3=0 && u.at.iDbnDb ); + assert( db->aDb[u.at.iDb].pBt!=0 ); + assert( (p->btreeMask & (((yDbMask)1)<aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta); + pOut->u.i = u.at.iMeta; break; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VNext P1 P2 * * * +/* Opcode: SetCookie P1 P2 P3 * * ** -** Advance virtual table P1 to the next row in its result set and -** jump to instruction P2. Or, if the virtual table has reached -** the end of its result set, then fall through to the next instruction. +** Write the content of register P3 (interpreted as an integer) +** into cookie number P2 of database P1. P2==1 is the schema version. +** P2==2 is the database format. P2==3 is the recommended pager cache +** size, and so forth. P1==0 is the main database file and P1==1 is the +** database file used to store temporary tables. +** +** A transaction must be started before executing this opcode. */ -case OP_VNext: { /* jump */ -#if 0 /* local variables moved into u.cj */ - sqlite3_vtab *pVtab; - const sqlite3_module *pModule; - int res; - VdbeCursor *pCur; -#endif /* local variables moved into u.cj */ - - u.cj.res = 0; - u.cj.pCur = p->apCsr[pOp->p1]; - assert( u.cj.pCur->pVtabCursor ); - if( u.cj.pCur->nullRow ){ - break; - } - u.cj.pVtab = u.cj.pCur->pVtabCursor->pVtab; - u.cj.pModule = u.cj.pVtab->pModule; - assert( u.cj.pModule->xNext ); - - /* Invoke the xNext() method of the module. There is no way for the - ** underlying implementation to return an error if one occurs during - ** xNext(). Instead, if an error occurs, true is returned (indicating that - ** data is available) and the error code returned when xColumn or - ** some other method is next invoked on the save virtual table cursor. - */ - p->inVtabMethod = 1; - rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor); - p->inVtabMethod = 0; - importVtabErrMsg(p, u.cj.pVtab); - if( rc==SQLITE_OK ){ - u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor); +case OP_SetCookie: { /* in3 */ +#if 0 /* local variables moved into u.au */ + Db *pDb; +#endif /* local variables moved into u.au */ + assert( pOp->p2p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + u.au.pDb = &db->aDb[pOp->p1]; + assert( u.au.pDb->pBt!=0 ); + assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); + pIn3 = &aMem[pOp->p3]; + sqlite3VdbeMemIntegerify(pIn3); + /* See note about index shifting on OP_ReadCookie */ + rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i); + if( pOp->p2==BTREE_SCHEMA_VERSION ){ + /* When the schema cookie changes, record the new cookie internally */ + u.au.pDb->pSchema->schema_cookie = (int)pIn3->u.i; + db->flags |= SQLITE_InternChanges; + }else if( pOp->p2==BTREE_FILE_FORMAT ){ + /* Record changes in the file format */ + u.au.pDb->pSchema->file_format = (u8)pIn3->u.i; } - - if( !u.cj.res ){ - /* If there is data, jump to P2 */ - pc = pOp->p2 - 1; + if( pOp->p1==1 ){ + /* Invalidate all prepared statements whenever the TEMP database + ** schema is changed. Ticket #1644 */ + sqlite3ExpirePreparedStatements(db); + p->expired = 0; } break; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VRename P1 * * P4 * +/* Opcode: VerifyCookie P1 P2 P3 * * ** -** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. -** This opcode invokes the corresponding xRename method. The value -** in register P1 is passed as the zName argument to the xRename method. +** Check the value of global database parameter number 0 (the +** schema version) and make sure it is equal to P2 and that the +** generation counter on the local schema parse equals P3. +** +** P1 is the database number which is 0 for the main database file +** and 1 for the file holding temporary tables and some higher number +** for auxiliary databases. +** +** The cookie changes its value whenever the database schema changes. +** This operation is used to detect when that the cookie has changed +** and that the current process needs to reread the schema. +** +** Either a transaction needs to have been started or an OP_Open needs +** to be executed (to establish a read lock) before this opcode is +** invoked. */ -case OP_VRename: { -#if 0 /* local variables moved into u.ck */ - sqlite3_vtab *pVtab; - Mem *pName; -#endif /* local variables moved into u.ck */ +case OP_VerifyCookie: { +#if 0 /* local variables moved into u.av */ + int iMeta; + int iGen; + Btree *pBt; +#endif /* local variables moved into u.av */ - u.ck.pVtab = pOp->p4.pVtab->pVtab; - u.ck.pName = &aMem[pOp->p1]; - assert( u.ck.pVtab->pModule->xRename ); - assert( memIsValid(u.ck.pName) ); - REGISTER_TRACE(pOp->p1, u.ck.pName); - assert( u.ck.pName->flags & MEM_Str ); - rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z); - importVtabErrMsg(p, u.ck.pVtab); - p->expired = 0; + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); + u.av.pBt = db->aDb[pOp->p1].pBt; + if( u.av.pBt ){ + sqlite3BtreeGetMeta(u.av.pBt, BTREE_SCHEMA_VERSION, (u32 *)&u.av.iMeta); + u.av.iGen = db->aDb[pOp->p1].pSchema->iGeneration; + }else{ + u.av.iGen = u.av.iMeta = 0; + } + if( u.av.iMeta!=pOp->p2 || u.av.iGen!=pOp->p3 ){ + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); + /* If the schema-cookie from the database file matches the cookie + ** stored with the in-memory representation of the schema, do + ** not reload the schema from the database file. + ** + ** If virtual-tables are in use, this is not just an optimization. + ** Often, v-tables store their data in other SQLite tables, which + ** are queried from within xNext() and other v-table methods using + ** prepared queries. If such a query is out-of-date, we do not want to + ** discard the database schema, as the user code implementing the + ** v-table would have to be ready for the sqlite3_vtab structure itself + ** to be invalidated whenever sqlite3_step() is called from within + ** a v-table method. + */ + if( db->aDb[pOp->p1].pSchema->schema_cookie!=u.av.iMeta ){ + sqlite3ResetInternalSchema(db, pOp->p1); + } + p->expired = 1; + rc = SQLITE_SCHEMA; + } break; } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VUpdate P1 P2 P3 P4 * +/* Opcode: OpenRead P1 P2 P3 P4 P5 ** -** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. -** This opcode invokes the corresponding xUpdate method. P2 values -** are contiguous memory cells starting at P3 to pass to the xUpdate -** invocation. The value in register (P3+P2-1) corresponds to the -** p2th element of the argv array passed to xUpdate. +** Open a read-only cursor for the database table whose root page is +** P2 in a database file. The database file is determined by P3. +** P3==0 means the main database, P3==1 means the database used for +** temporary tables, and P3>1 means used the corresponding attached +** database. Give the new cursor an identifier of P1. The P1 +** values need not be contiguous but all P1 values should be small integers. +** It is an error for P1 to be negative. ** -** The xUpdate method will do a DELETE or an INSERT or both. -** The argv[0] element (which corresponds to memory cell P3) -** is the rowid of a row to delete. If argv[0] is NULL then no -** deletion occurs. The argv[1] element is the rowid of the new -** row. This can be NULL to have the virtual table select the new -** rowid for itself. The subsequent elements in the array are -** the values of columns in the new row. +** If P5!=0 then use the content of register P2 as the root page, not +** the value of P2 itself. ** -** If P2==1 then no insert is performed. argv[0] is the rowid of -** a row to delete. +** There will be a read lock on the database whenever there is an +** open cursor. If the database was unlocked prior to this instruction +** then a read lock is acquired as part of this instruction. A read +** lock allows other processes to read the database but prohibits +** any other process from modifying the database. The read lock is +** released when all cursors are closed. If this instruction attempts +** to get a read lock but fails, the script terminates with an +** SQLITE_BUSY error code. ** -** P1 is a boolean flag. If it is set to true and the xUpdate call -** is successful, then the value returned by sqlite3_last_insert_rowid() -** is set to the value of the rowid for the row just inserted. +** The P4 value may be either an integer (P4_INT32) or a pointer to +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** structure, then said structure defines the content and collating +** sequence of the index being opened. Otherwise, if P4 is an integer +** value, it is set to the number of columns in the table. +** +** See also OpenWrite. */ -case OP_VUpdate: { -#if 0 /* local variables moved into u.cl */ - sqlite3_vtab *pVtab; - sqlite3_module *pModule; - int nArg; - int i; - sqlite_int64 rowid; - Mem **apArg; - Mem *pX; -#endif /* local variables moved into u.cl */ +/* Opcode: OpenWrite P1 P2 P3 P4 P5 +** +** Open a read/write cursor named P1 on the table or index whose root +** page is P2. Or if P5!=0 use the content of register P2 to find the +** root page. +** +** The P4 value may be either an integer (P4_INT32) or a pointer to +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** structure, then said structure defines the content and collating +** sequence of the index being opened. Otherwise, if P4 is an integer +** value, it is set to the number of columns in the table, or to the +** largest index of any column of the table that is actually used. +** +** This instruction works just like OpenRead except that it opens the cursor +** in read/write mode. For a given table, there can be one or more read-only +** cursors or a single read/write cursor but not both. +** +** See also OpenRead. +*/ +case OP_OpenRead: +case OP_OpenWrite: { +#if 0 /* local variables moved into u.aw */ + int nField; + KeyInfo *pKeyInfo; + int p2; + int iDb; + int wrFlag; + Btree *pX; + VdbeCursor *pCur; + Db *pDb; +#endif /* local variables moved into u.aw */ - u.cl.pVtab = pOp->p4.pVtab->pVtab; - u.cl.pModule = (sqlite3_module *)u.cl.pVtab->pModule; - u.cl.nArg = pOp->p2; - assert( pOp->p4type==P4_VTAB ); - if( ALWAYS(u.cl.pModule->xUpdate) ){ - u.cl.apArg = p->apArg; - u.cl.pX = &aMem[pOp->p3]; - for(u.cl.i=0; u.cl.ixUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid); - importVtabErrMsg(p, u.cl.pVtab); - if( rc==SQLITE_OK && pOp->p1 ){ - assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) ); - db->lastRowid = u.cl.rowid; + if( p->expired ){ + rc = SQLITE_ABORT; + break; + } + + u.aw.nField = 0; + u.aw.pKeyInfo = 0; + u.aw.p2 = pOp->p2; + u.aw.iDb = pOp->p3; + assert( u.aw.iDb>=0 && u.aw.iDbnDb ); + assert( (p->btreeMask & (((yDbMask)1)<aDb[u.aw.iDb]; + u.aw.pX = u.aw.pDb->pBt; + assert( u.aw.pX!=0 ); + if( pOp->opcode==OP_OpenWrite ){ + u.aw.wrFlag = 1; + assert( sqlite3SchemaMutexHeld(db, u.aw.iDb, 0) ); + if( u.aw.pDb->pSchema->file_format < p->minWriteFileFormat ){ + p->minWriteFileFormat = u.aw.pDb->pSchema->file_format; } - p->nChange++; + }else{ + u.aw.wrFlag = 0; } - break; -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ + if( pOp->p5 ){ + assert( u.aw.p2>0 ); + assert( u.aw.p2<=p->nMem ); + pIn2 = &aMem[u.aw.p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); + sqlite3VdbeMemIntegerify(pIn2); + u.aw.p2 = (int)pIn2->u.i; + /* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and + ** that opcode will always set the u.aw.p2 value to 2 or more or else fail. + ** If there were a failure, the prepared statement would have halted + ** before reaching this instruction. */ + if( NEVER(u.aw.p2<2) ) { + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } + } + if( pOp->p4type==P4_KEYINFO ){ + u.aw.pKeyInfo = pOp->p4.pKeyInfo; + u.aw.pKeyInfo->enc = ENC(p->db); + u.aw.nField = u.aw.pKeyInfo->nField+1; + }else if( pOp->p4type==P4_INT32 ){ + u.aw.nField = pOp->p4.i; + } + assert( pOp->p1>=0 ); + u.aw.pCur = allocateCursor(p, pOp->p1, u.aw.nField, u.aw.iDb, 1); + if( u.aw.pCur==0 ) goto no_mem; + u.aw.pCur->nullRow = 1; + u.aw.pCur->isOrdered = 1; + rc = sqlite3BtreeCursor(u.aw.pX, u.aw.p2, u.aw.wrFlag, u.aw.pKeyInfo, u.aw.pCur->pCursor); + u.aw.pCur->pKeyInfo = u.aw.pKeyInfo; -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* Opcode: Pagecount P1 P2 * * * -** -** Write the current number of pages in database P1 to memory cell P2. -*/ -case OP_Pagecount: { /* out2-prerelease */ - pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); + /* Since it performs no memory allocation or IO, the only values that + ** sqlite3BtreeCursor() may return are SQLITE_EMPTY and SQLITE_OK. + ** SQLITE_EMPTY is only returned when attempting to open the table + ** rooted at page 1 of a zero-byte database. */ + assert( rc==SQLITE_EMPTY || rc==SQLITE_OK ); + if( rc==SQLITE_EMPTY ){ + u.aw.pCur->pCursor = 0; + rc = SQLITE_OK; + } + + /* Set the VdbeCursor.isTable and isIndex variables. Previous versions of + ** SQLite used to check if the root-page flags were sane at this point + ** and report database corruption if they were not, but this check has + ** since moved into the btree layer. */ + u.aw.pCur->isTable = pOp->p4type!=P4_KEYINFO; + u.aw.pCur->isIndex = !u.aw.pCur->isTable; break; } -#endif - -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* Opcode: MaxPgcnt P1 P2 P3 * * +/* Opcode: OpenEphemeral P1 P2 * P4 * ** -** Try to set the maximum page count for database P1 to the value in P3. -** Do not let the maximum page count fall below the current page count and -** do not change the maximum page count value if P3==0. +** Open a new cursor P1 to a transient table. +** The cursor is always opened read/write even if +** the main database is read-only. The ephemeral +** table is deleted automatically when the cursor is closed. ** -** Store the maximum page count after the change in register P2. +** P2 is the number of columns in the ephemeral table. +** The cursor points to a BTree table if P4==0 and to a BTree index +** if P4 is not 0. If P4 is not NULL, it points to a KeyInfo structure +** that defines the format of keys in the index. +** +** This opcode was once called OpenTemp. But that created +** confusion because the term "temp table", might refer either +** to a TEMP table at the SQL level, or to a table opened by +** this opcode. Then this opcode was call OpenVirtual. But +** that created confusion with the whole virtual-table idea. */ -case OP_MaxPgcnt: { /* out2-prerelease */ - unsigned int newMax; - Btree *pBt; +/* Opcode: OpenAutoindex P1 P2 * P4 * +** +** This opcode works the same as OP_OpenEphemeral. It has a +** different name to distinguish its use. Tables created using +** by this opcode will be used for automatically created transient +** indices in joins. +*/ +case OP_OpenAutoindex: +case OP_OpenEphemeral: { +#if 0 /* local variables moved into u.ax */ + VdbeCursor *pCx; +#endif /* local variables moved into u.ax */ + static const int vfsFlags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TRANSIENT_DB; - pBt = db->aDb[pOp->p1].pBt; - newMax = 0; - if( pOp->p3 ){ - newMax = sqlite3BtreeLastPage(pBt); - if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3; + assert( pOp->p1>=0 ); + u.ax.pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + if( u.ax.pCx==0 ) goto no_mem; + u.ax.pCx->nullRow = 1; + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &u.ax.pCx->pBt, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(u.ax.pCx->pBt, 1); } - pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); + if( rc==SQLITE_OK ){ + /* If a transient index is required, create it by calling + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before + ** opening it. If a transient table is required, just use the + ** automatically created table with root-page 1 (an BLOB_INTKEY table). + */ + if( pOp->p4.pKeyInfo ){ + int pgno; + assert( pOp->p4type==P4_KEYINFO ); + rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY); + if( rc==SQLITE_OK ){ + assert( pgno==MASTER_ROOT+1 ); + rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1, + (KeyInfo*)pOp->p4.z, u.ax.pCx->pCursor); + u.ax.pCx->pKeyInfo = pOp->p4.pKeyInfo; + u.ax.pCx->pKeyInfo->enc = ENC(p->db); + } + u.ax.pCx->isTable = 0; + }else{ + rc = sqlite3BtreeCursor(u.ax.pCx->pBt, MASTER_ROOT, 1, 0, u.ax.pCx->pCursor); + u.ax.pCx->isTable = 1; + } + } + u.ax.pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + u.ax.pCx->isIndex = !u.ax.pCx->isTable; break; } -#endif - -#ifndef SQLITE_OMIT_TRACE -/* Opcode: Trace * * * P4 * +/* Opcode: OpenPseudo P1 P2 P3 * * ** -** If tracing is enabled (by the sqlite3_trace()) interface, then -** the UTF-8 string contained in P4 is emitted on the trace callback. +** Open a new cursor that points to a fake table that contains a single +** row of data. The content of that one row in the content of memory +** register P2. In other words, cursor P1 becomes an alias for the +** MEM_Blob content contained in register P2. +** +** A pseudo-table created by this opcode is used to hold a single +** row output from the sorter so that the row can be decomposed into +** individual columns using the OP_Column opcode. The OP_Column opcode +** is the only cursor opcode that works with a pseudo-table. +** +** P3 is the number of fields in the records that will be stored by +** the pseudo-table. */ -case OP_Trace: { -#if 0 /* local variables moved into u.cm */ - char *zTrace; -#endif /* local variables moved into u.cm */ +case OP_OpenPseudo: { +#if 0 /* local variables moved into u.ay */ + VdbeCursor *pCx; +#endif /* local variables moved into u.ay */ - u.cm.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); - if( u.cm.zTrace ){ - if( db->xTrace ){ - char *z = sqlite3VdbeExpandSql(p, u.cm.zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); - } -#ifdef SQLITE_DEBUG - if( (db->flags & SQLITE_SqlTrace)!=0 ){ - sqlite3DebugPrintf("SQL-trace: %s\n", u.cm.zTrace); - } -#endif /* SQLITE_DEBUG */ - } + assert( pOp->p1>=0 ); + u.ay.pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0); + if( u.ay.pCx==0 ) goto no_mem; + u.ay.pCx->nullRow = 1; + u.ay.pCx->pseudoTableReg = pOp->p2; + u.ay.pCx->isTable = 1; + u.ay.pCx->isIndex = 0; break; } -#endif - -/* Opcode: Noop * * * * * +/* Opcode: Close P1 * * * * ** -** Do nothing. This instruction is often useful as a jump -** destination. -*/ -/* -** The magic Explain opcode are only inserted when explain==2 (which -** is to say when the EXPLAIN QUERY PLAN syntax is used.) -** This opcode records information from the optimizer. It is the -** the same as a no-op. This opcodesnever appears in a real VM program. +** Close a cursor previously opened as P1. If P1 is not +** currently open, this instruction is a no-op. */ -default: { /* This is really OP_Noop and OP_Explain */ - assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); +case OP_Close: { + assert( pOp->p1>=0 && pOp->p1nCursor ); + sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); + p->apCsr[pOp->p1] = 0; break; } -/***************************************************************************** -** The cases of the switch statement above this line should all be indented -** by 6 spaces. But the left-most 6 spaces have been removed to improve the -** readability. From this point on down, the normal indentation rules are -** restored. -*****************************************************************************/ - } - -#ifdef VDBE_PROFILE - { - u64 elapsed = sqlite3Hwtime() - start; - pOp->cycles += elapsed; - pOp->cnt++; -#if 0 - fprintf(stdout, "%10llu ", elapsed); - sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]); -#endif - } -#endif - - /* The following code adds nothing to the actual functionality - ** of the program. It is only here for testing and debugging. - ** On the other hand, it does burn CPU cycles every time through - ** the evaluator loop. So we can leave it out when NDEBUG is defined. - */ -#ifndef NDEBUG - assert( pc>=-1 && pcnOp ); - -#ifdef SQLITE_DEBUG - if( p->trace ){ - if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc); - if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ - registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]); - } - if( pOp->opflags & OPFLG_OUT3 ){ - registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]); - } - } -#endif /* SQLITE_DEBUG */ -#endif /* NDEBUG */ - } /* The end of the for(;;) loop the loops through opcodes */ - - /* If we reach this point, it means that execution is finished with - ** an error of some kind. - */ -vdbe_error_halt: - assert( rc ); - p->rc = rc; - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: [%s] %s", - pc, p->zSql, p->zErrMsg); - sqlite3VdbeHalt(p); - if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; - rc = SQLITE_ERROR; - if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0); - - /* This is the only way out of this procedure. We have to - ** release the mutexes on btrees that were acquired at the - ** top. */ -vdbe_return: - sqlite3BtreeMutexArrayLeave(&p->aMutex); - return rc; - - /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH - ** is encountered. - */ -too_big: - sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); - rc = SQLITE_TOOBIG; - goto vdbe_error_halt; - - /* Jump to here if a malloc() fails. - */ -no_mem: - db->mallocFailed = 1; - sqlite3SetString(&p->zErrMsg, db, "out of memory"); - rc = SQLITE_NOMEM; - goto vdbe_error_halt; - - /* Jump to here for any other kind of fatal error. The "rc" variable - ** should hold the error number. - */ -abort_due_to_error: - assert( p->zErrMsg==0 ); - if( db->mallocFailed ) rc = SQLITE_NOMEM; - if( rc!=SQLITE_IOERR_NOMEM ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); - } - goto vdbe_error_halt; - - /* Jump to here if the sqlite3_interrupt() API sets the interrupt - ** flag. - */ -abort_due_to_interrupt: - assert( db->u1.isInterrupted ); - rc = SQLITE_INTERRUPT; - p->rc = rc; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); - goto vdbe_error_halt; -} - -/************** End of vdbe.c ************************************************/ -/************** Begin file vdbeblob.c ****************************************/ -/* -** 2007 May 1 +/* Opcode: SeekGe P1 P2 P3 P4 * ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as the key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than or equal to the key value. If there are no records +** greater than or equal to the key and P2 is not zero, then jump to P2. ** -************************************************************************* +** See also: Found, NotFound, Distinct, SeekLt, SeekGt, SeekLe +*/ +/* Opcode: SeekGt P1 P2 P3 P4 * ** -** This file contains code used to implement incremental BLOB I/O. +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than the key value. If there are no records greater than +** the key and P2 is not zero, then jump to P2. +** +** See also: Found, NotFound, Distinct, SeekLt, SeekGe, SeekLe */ - - -#ifndef SQLITE_OMIT_INCRBLOB - -/* -** Valid sqlite3_blob* handles point to Incrblob structures. +/* Opcode: SeekLt P1 P2 P3 P4 * +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the largest entry that +** is less than the key value. If there are no records less than +** the key and P2 is not zero, then jump to P2. +** +** See also: Found, NotFound, Distinct, SeekGt, SeekGe, SeekLe */ -typedef struct Incrblob Incrblob; -struct Incrblob { - int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ - int nByte; /* Size of open blob, in bytes */ - int iOffset; /* Byte offset of blob in cursor data */ - int iCol; /* Table column this handle is open on */ - BtCursor *pCsr; /* Cursor pointing at blob row */ - sqlite3_stmt *pStmt; /* Statement holding cursor open */ - sqlite3 *db; /* The associated database */ -}; - - -/* -** This function is used by both blob_open() and blob_reopen(). It seeks -** the b-tree cursor associated with blob handle p to point to row iRow. -** If successful, SQLITE_OK is returned and subsequent calls to -** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. +/* Opcode: SeekLe P1 P2 P3 P4 * ** -** If an error occurs, or if the specified row does not exist or does not -** contain a value of type TEXT or BLOB in the column nominated when the -** blob handle was opened, then an error code is returned and *pzErr may -** be set to point to a buffer containing an error message. It is the -** responsibility of the caller to free the error message buffer using -** sqlite3DbFree(). +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. ** -** If an error does occur, then the b-tree cursor is closed. All subsequent -** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will -** immediately return SQLITE_ABORT. +** Reposition cursor P1 so that it points to the largest entry that +** is less than or equal to the key value. If there are no records +** less than or equal to the key and P2 is not zero, then jump to P2. +** +** See also: Found, NotFound, Distinct, SeekGt, SeekGe, SeekLt */ -static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ - int rc; /* Error code */ - char *zErr = 0; /* Error message */ - Vdbe *v = (Vdbe *)p->pStmt; - - /* Set the value of the SQL statements only variable to integer iRow. - ** This is done directly instead of using sqlite3_bind_int64() to avoid - ** triggering asserts related to mutexes. - */ - assert( v->aVar[0].flags&MEM_Int ); - v->aVar[0].u.i = iRow; +case OP_SeekLt: /* jump, in3 */ +case OP_SeekLe: /* jump, in3 */ +case OP_SeekGe: /* jump, in3 */ +case OP_SeekGt: { /* jump, in3 */ +#if 0 /* local variables moved into u.az */ + int res; + int oc; + VdbeCursor *pC; + UnpackedRecord r; + int nField; + i64 iKey; /* The rowid we are to seek to */ +#endif /* local variables moved into u.az */ - rc = sqlite3_step(p->pStmt); - if( rc==SQLITE_ROW ){ - u32 type = v->apCsr[0]->aType[p->iCol]; - if( type<12 ){ - zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", - type==0?"null": type==7?"real": "integer" - ); - rc = SQLITE_ERROR; - sqlite3_finalize(p->pStmt); - p->pStmt = 0; - }else{ - p->iOffset = v->apCsr[0]->aOffset[p->iCol]; - p->nByte = sqlite3VdbeSerialTypeLen(type); - p->pCsr = v->apCsr[0]->pCursor; - sqlite3BtreeEnterCursor(p->pCsr); - sqlite3BtreeCacheOverflow(p->pCsr); - sqlite3BtreeLeaveCursor(p->pCsr); - } - } - - if( rc==SQLITE_ROW ){ - rc = SQLITE_OK; - }else if( p->pStmt ){ - rc = sqlite3_finalize(p->pStmt); - p->pStmt = 0; - if( rc==SQLITE_OK ){ - zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); - rc = SQLITE_ERROR; - }else{ - zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); - } - } - - assert( rc!=SQLITE_OK || zErr==0 ); - assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); - - *pzErr = zErr; - return rc; -} - -/* -** Open a blob handle. -*/ -SQLITE_API int sqlite3_blob_open( - sqlite3* db, /* The database connection */ - const char *zDb, /* The attached database containing the blob */ - const char *zTable, /* The table containing the blob */ - const char *zColumn, /* The column containing the blob */ - sqlite_int64 iRow, /* The row containing the glob */ - int flags, /* True -> read/write access, false -> read-only */ - sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ -){ - int nAttempt = 0; - int iCol; /* Index of zColumn in row-record */ - - /* This VDBE program seeks a btree cursor to the identified - ** db/table/row entry. The reason for using a vdbe program instead - ** of writing code to use the b-tree layer directly is that the - ** vdbe program will take advantage of the various transaction, - ** locking and error handling infrastructure built into the vdbe. - ** - ** After seeking the cursor, the vdbe executes an OP_ResultRow. - ** Code external to the Vdbe then "borrows" the b-tree cursor and - ** uses it to implement the blob_read(), blob_write() and - ** blob_bytes() functions. - ** - ** The sqlite3_blob_close() function finalizes the vdbe program, - ** which closes the b-tree cursor and (possibly) commits the - ** transaction. - */ - static const VdbeOpList openBlob[] = { - {OP_Transaction, 0, 0, 0}, /* 0: Start a transaction */ - {OP_VerifyCookie, 0, 0, 0}, /* 1: Check the schema cookie */ - {OP_TableLock, 0, 0, 0}, /* 2: Acquire a read or write lock */ - - /* One of the following two instructions is replaced by an OP_Noop. */ - {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ - {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ - - {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ - {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 7 */ - {OP_ResultRow, 1, 0, 0}, /* 8 */ - {OP_Goto, 0, 5, 0}, /* 9 */ - {OP_Close, 0, 0, 0}, /* 10 */ - {OP_Halt, 0, 0, 0}, /* 11 */ - }; - - int rc = SQLITE_OK; - char *zErr = 0; - Table *pTab; - Parse *pParse = 0; - Incrblob *pBlob = 0; - - flags = !!flags; /* flags = (flags ? 1 : 0); */ - *ppBlob = 0; - - sqlite3_mutex_enter(db->mutex); - - pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - if( !pBlob ) goto blob_open_out; - pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( !pParse ) goto blob_open_out; - - do { - memset(pParse, 0, sizeof(Parse)); - pParse->db = db; - sqlite3DbFree(db, zErr); - zErr = 0; - - sqlite3BtreeEnterAll(db); - pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); - if( pTab && IsVirtual(pTab) ){ - pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); - } -#ifndef SQLITE_OMIT_VIEW - if( pTab && pTab->pSelect ){ - pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open view: %s", zTable); - } -#endif - if( !pTab ){ - if( pParse->zErrMsg ){ - sqlite3DbFree(db, zErr); - zErr = pParse->zErrMsg; - pParse->zErrMsg = 0; - } - rc = SQLITE_ERROR; - sqlite3BtreeLeaveAll(db); - goto blob_open_out; - } + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p2!=0 ); + u.az.pC = p->apCsr[pOp->p1]; + assert( u.az.pC!=0 ); + assert( u.az.pC->pseudoTableReg==0 ); + assert( OP_SeekLe == OP_SeekLt+1 ); + assert( OP_SeekGe == OP_SeekLt+2 ); + assert( OP_SeekGt == OP_SeekLt+3 ); + assert( u.az.pC->isOrdered ); + if( u.az.pC->pCursor!=0 ){ + u.az.oc = pOp->opcode; + u.az.pC->nullRow = 0; + if( u.az.pC->isTable ){ + /* The input value in P3 might be of any type: integer, real, string, + ** blob, or NULL. But it needs to be an integer before we can do + ** the seek, so covert it. */ + pIn3 = &aMem[pOp->p3]; + applyNumericAffinity(pIn3); + u.az.iKey = sqlite3VdbeIntValue(pIn3); + u.az.pC->rowidIsValid = 0; - /* Now search pTab for the exact column. */ - for(iCol=0; iColnCol; iCol++) { - if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ - break; - } - } - if( iCol==pTab->nCol ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); - rc = SQLITE_ERROR; - sqlite3BtreeLeaveAll(db); - goto blob_open_out; - } + /* If the P3 value could not be converted into an integer without + ** loss of information, then special processing is required... */ + if( (pIn3->flags & MEM_Int)==0 ){ + if( (pIn3->flags & MEM_Real)==0 ){ + /* If the P3 value cannot be converted into any kind of a number, + ** then the seek is not possible, so jump to P2 */ + pc = pOp->p2 - 1; + break; + } + /* If we reach this point, then the P3 value must be a floating + ** point number. */ + assert( (pIn3->flags & MEM_Real)!=0 ); - /* If the value is being opened for writing, check that the - ** column is not indexed, and that it is not part of a foreign key. - ** It is against the rules to open a column to which either of these - ** descriptions applies for writing. */ - if( flags ){ - const char *zFault = 0; - Index *pIdx; -#ifndef SQLITE_OMIT_FOREIGN_KEY - if( db->flags&SQLITE_ForeignKeys ){ - /* Check that the column is not part of an FK child key definition. It - ** is not necessary to check if it is part of a parent key, as parent - ** key columns must be indexed. The check below will pick up this - ** case. */ - FKey *pFKey; - for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ - int j; - for(j=0; jnCol; j++){ - if( pFKey->aCol[j].iFrom==iCol ){ - zFault = "foreign key"; + if( u.az.iKey==SMALLEST_INT64 && (pIn3->r<(double)u.az.iKey || pIn3->r>0) ){ + /* The P3 value is too large in magnitude to be expressed as an + ** integer. */ + u.az.res = 1; + if( pIn3->r<0 ){ + if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt ); + rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } + }else{ + if( u.az.oc<=OP_SeekLe ){ assert( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe ); + rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } } - } - } -#endif - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int j; - for(j=0; jnColumn; j++){ - if( pIdx->aiColumn[j]==iCol ){ - zFault = "indexed"; + if( u.az.res ){ + pc = pOp->p2 - 1; } + break; + }else if( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekGe ){ + /* Use the ceiling() function to convert real->int */ + if( pIn3->r > (double)u.az.iKey ) u.az.iKey++; + }else{ + /* Use the floor() function to convert real->int */ + assert( u.az.oc==OP_SeekLe || u.az.oc==OP_SeekGt ); + if( pIn3->r < (double)u.az.iKey ) u.az.iKey--; } } - if( zFault ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); - rc = SQLITE_ERROR; - sqlite3BtreeLeaveAll(db); - goto blob_open_out; + rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, 0, (u64)u.az.iKey, 0, &u.az.res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; } - } - - pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db); - assert( pBlob->pStmt || db->mallocFailed ); - if( pBlob->pStmt ){ - Vdbe *v = (Vdbe *)pBlob->pStmt; - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - - sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); - - - /* Configure the OP_Transaction */ - sqlite3VdbeChangeP1(v, 0, iDb); - sqlite3VdbeChangeP2(v, 0, flags); - - /* Configure the OP_VerifyCookie */ - sqlite3VdbeChangeP1(v, 1, iDb); - sqlite3VdbeChangeP2(v, 1, pTab->pSchema->schema_cookie); + if( u.az.res==0 ){ + u.az.pC->rowidIsValid = 1; + u.az.pC->lastRowid = u.az.iKey; + } + }else{ + u.az.nField = pOp->p4.i; + assert( pOp->p4type==P4_INT32 ); + assert( u.az.nField>0 ); + u.az.r.pKeyInfo = u.az.pC->pKeyInfo; + u.az.r.nField = (u16)u.az.nField; - /* Make sure a mutex is held on the table to be accessed */ - sqlite3VdbeUsesBtree(v, iDb); + /* The next line of code computes as follows, only faster: + ** if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekLe ){ + ** u.az.r.flags = UNPACKED_INCRKEY; + ** }else{ + ** u.az.r.flags = 0; + ** } + */ + u.az.r.flags = (u16)(UNPACKED_INCRKEY * (1 & (u.az.oc - OP_SeekLt))); + assert( u.az.oc!=OP_SeekGt || u.az.r.flags==UNPACKED_INCRKEY ); + assert( u.az.oc!=OP_SeekLe || u.az.r.flags==UNPACKED_INCRKEY ); + assert( u.az.oc!=OP_SeekGe || u.az.r.flags==0 ); + assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 ); - /* Configure the OP_TableLock instruction */ -#ifdef SQLITE_OMIT_SHARED_CACHE - sqlite3VdbeChangeToNoop(v, 2, 1); -#else - sqlite3VdbeChangeP1(v, 2, iDb); - sqlite3VdbeChangeP2(v, 2, pTab->tnum); - sqlite3VdbeChangeP3(v, 2, flags); - sqlite3VdbeChangeP4(v, 2, pTab->zName, P4_TRANSIENT); + u.az.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; itnum. */ - sqlite3VdbeChangeToNoop(v, 4 - flags, 1); - sqlite3VdbeChangeP2(v, 3 + flags, pTab->tnum); - sqlite3VdbeChangeP3(v, 3 + flags, iDb); - - /* Configure the number of columns. Configure the cursor to - ** think that the table has one more column than it really - ** does. An OP_Column to retrieve this imaginary column will - ** always return an SQL NULL. This is useful because it means - ** we can invoke OP_Column to fill in the vdbe cursors type - ** and offset cache without causing any IO. - */ - sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); - sqlite3VdbeChangeP2(v, 7, pTab->nCol); - if( !db->mallocFailed ){ - sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); + ExpandBlob(u.az.r.aMem); + rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, &u.az.r, 0, 0, &u.az.res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; } + u.az.pC->rowidIsValid = 0; } - - pBlob->flags = flags; - pBlob->iCol = iCol; - pBlob->db = db; - sqlite3BtreeLeaveAll(db); - if( db->mallocFailed ){ - goto blob_open_out; + u.az.pC->deferredMoveto = 0; + u.az.pC->cacheStatus = CACHE_STALE; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt ); + if( u.az.res<0 || (u.az.res==0 && u.az.oc==OP_SeekGt) ){ + rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + u.az.pC->rowidIsValid = 0; + }else{ + u.az.res = 0; + } + }else{ + assert( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe ); + if( u.az.res>0 || (u.az.res==0 && u.az.oc==OP_SeekLt) ){ + rc = sqlite3BtreePrevious(u.az.pC->pCursor, &u.az.res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + u.az.pC->rowidIsValid = 0; + }else{ + /* u.az.res might be negative because the table is empty. Check to + ** see if this is the case. + */ + u.az.res = sqlite3BtreeEof(u.az.pC->pCursor); + } + } + assert( pOp->p2>0 ); + if( u.az.res ){ + pc = pOp->p2 - 1; } - sqlite3_bind_int64(pBlob->pStmt, 1, iRow); - rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); - -blob_open_out: - if( rc==SQLITE_OK && db->mallocFailed==0 ){ - *ppBlob = (sqlite3_blob *)pBlob; }else{ - if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); - sqlite3DbFree(db, pBlob); + /* This happens when attempting to open the sqlite3_master table + ** for read access returns SQLITE_EMPTY. In this case always + ** take the jump (since there are no records in the table). + */ + pc = pOp->p2 - 1; } - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); - sqlite3DbFree(db, zErr); - sqlite3StackFree(db, pParse); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + break; } -/* -** Close a blob handle that was previously created using -** sqlite3_blob_open(). +/* Opcode: Seek P1 P2 * * * +** +** P1 is an open table cursor and P2 is a rowid integer. Arrange +** for P1 to move so that it points to the rowid given by P2. +** +** This is actually a deferred seek. Nothing actually happens until +** the cursor is used to read a record. That way, if no reads +** occur, no unnecessary I/O happens. */ -SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ - Incrblob *p = (Incrblob *)pBlob; - int rc; - sqlite3 *db; +case OP_Seek: { /* in2 */ +#if 0 /* local variables moved into u.ba */ + VdbeCursor *pC; +#endif /* local variables moved into u.ba */ - if( p ){ - db = p->db; - sqlite3_mutex_enter(db->mutex); - rc = sqlite3_finalize(p->pStmt); - sqlite3DbFree(db, p); - sqlite3_mutex_leave(db->mutex); - }else{ - rc = SQLITE_OK; + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.ba.pC = p->apCsr[pOp->p1]; + assert( u.ba.pC!=0 ); + if( ALWAYS(u.ba.pC->pCursor!=0) ){ + assert( u.ba.pC->isTable ); + u.ba.pC->nullRow = 0; + pIn2 = &aMem[pOp->p2]; + u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2); + u.ba.pC->rowidIsValid = 0; + u.ba.pC->deferredMoveto = 1; } - return rc; + break; } + -/* -** Perform a read or write operation on a blob +/* Opcode: Found P1 P2 P3 P4 * +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is a prefix of any entry in P1 then a jump is made to P2 and +** P1 is left pointing at the matching entry. */ -static int blobReadWrite( - sqlite3_blob *pBlob, - void *z, - int n, - int iOffset, - int (*xCall)(BtCursor*, u32, u32, void*) -){ - int rc; - Incrblob *p = (Incrblob *)pBlob; - Vdbe *v; - sqlite3 *db; +/* Opcode: NotFound P1 P2 P3 P4 * +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is not the prefix of any entry in P1 then a jump is made to P2. If P1 +** does contain an entry whose prefix matches the P3/P4 record then control +** falls through to the next instruction and P1 is left pointing at the +** matching entry. +** +** See also: Found, NotExists, IsUnique +*/ +case OP_NotFound: /* jump, in3 */ +case OP_Found: { /* jump, in3 */ +#if 0 /* local variables moved into u.bb */ + int alreadyExists; + VdbeCursor *pC; + int res; + UnpackedRecord *pIdxKey; + UnpackedRecord r; + char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; +#endif /* local variables moved into u.bb */ - if( p==0 ) return SQLITE_MISUSE_BKPT; - db = p->db; - sqlite3_mutex_enter(db->mutex); - v = (Vdbe*)p->pStmt; +#ifdef SQLITE_TEST + sqlite3_found_count++; +#endif - if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ - /* Request is out of range. Return a transient error. */ - rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, 0); - }else if( v==0 ){ - /* If there is no statement handle, then the blob-handle has - ** already been invalidated. Return SQLITE_ABORT in this case. - */ - rc = SQLITE_ABORT; - }else{ - /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is - ** returned, clean-up the statement handle. - */ - assert( db == v->db ); - sqlite3BtreeEnterCursor(p->pCsr); - rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); - sqlite3BtreeLeaveCursor(p->pCsr); - if( rc==SQLITE_ABORT ){ - sqlite3VdbeFinalize(v); - p->pStmt = 0; + u.bb.alreadyExists = 0; + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + u.bb.pC = p->apCsr[pOp->p1]; + assert( u.bb.pC!=0 ); + pIn3 = &aMem[pOp->p3]; + if( ALWAYS(u.bb.pC->pCursor!=0) ){ + + assert( u.bb.pC->isTable==0 ); + if( pOp->p4.i>0 ){ + u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo; + u.bb.r.nField = (u16)pOp->p4.i; + u.bb.r.aMem = pIn3; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ierrCode = rc; - v->rc = rc; + assert( pIn3->flags & MEM_Blob ); + assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z, + u.bb.aTempRec, sizeof(u.bb.aTempRec)); + if( u.bb.pIdxKey==0 ){ + goto no_mem; + } + u.bb.pIdxKey->flags |= UNPACKED_PREFIX_MATCH; + } + rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res); + if( pOp->p4.i==0 ){ + sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey); + } + if( rc!=SQLITE_OK ){ + break; } + u.bb.alreadyExists = (u.bb.res==0); + u.bb.pC->deferredMoveto = 0; + u.bb.pC->cacheStatus = CACHE_STALE; } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + if( pOp->opcode==OP_Found ){ + if( u.bb.alreadyExists ) pc = pOp->p2 - 1; + }else{ + if( !u.bb.alreadyExists ) pc = pOp->p2 - 1; + } + break; } -/* -** Read data from a blob handle. -*/ -SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ - return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData); -} - -/* -** Write data to a blob handle. -*/ -SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ - return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); -} - -/* -** Query a blob handle for the size of the data. -** -** The Incrblob.nByte field is fixed for the lifetime of the Incrblob -** so no mutex is required for access. -*/ -SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ - Incrblob *p = (Incrblob *)pBlob; - return (p && p->pStmt) ? p->nByte : 0; -} - -/* -** Move an existing blob handle to point to a different row of the same -** database table. -** -** If an error occurs, or if the specified row does not exist or does not -** contain a blob or text value, then an error code is returned and the -** database handle error code and message set. If this happens, then all -** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) -** immediately return SQLITE_ABORT. -*/ -SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ - int rc; - Incrblob *p = (Incrblob *)pBlob; - sqlite3 *db; - - if( p==0 ) return SQLITE_MISUSE_BKPT; - db = p->db; - sqlite3_mutex_enter(db->mutex); - - if( p->pStmt==0 ){ - /* If there is no statement handle, then the blob-handle has - ** already been invalidated. Return SQLITE_ABORT in this case. - */ - rc = SQLITE_ABORT; - }else{ - char *zErr; - rc = blobSeekToRow(p, iRow, &zErr); - if( rc!=SQLITE_OK ){ - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); - sqlite3DbFree(db, zErr); - } - assert( rc!=SQLITE_SCHEMA ); - } - - rc = sqlite3ApiExit(db, rc); - assert( rc==SQLITE_OK || p->pStmt==0 ); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -#endif /* #ifndef SQLITE_OMIT_INCRBLOB */ - -/************** End of vdbeblob.c ********************************************/ -/************** Begin file journal.c *****************************************/ -/* -** 2007 August 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +/* Opcode: IsUnique P1 P2 P3 P4 * ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Cursor P1 is open on an index b-tree - that is to say, a btree which +** no data and where the key are records generated by OP_MakeRecord with +** the list field being the integer ROWID of the entry that the index +** entry refers to. ** -************************************************************************* +** The P3 register contains an integer record number. Call this record +** number R. Register P4 is the first in a set of N contiguous registers +** that make up an unpacked index key that can be used with cursor P1. +** The value of N can be inferred from the cursor. N includes the rowid +** value appended to the end of the index record. This rowid value may +** or may not be the same as R. ** -** This file implements a special kind of sqlite3_file object used -** by SQLite to create journal files if the atomic-write optimization -** is enabled. +** If any of the N registers beginning with register P4 contains a NULL +** value, jump immediately to P2. ** -** The distinctive characteristic of this sqlite3_file is that the -** actual on disk file is created lazily. When the file is created, -** the caller specifies a buffer size for an in-memory buffer to -** be used to service read() and write() requests. The actual file -** on disk is not created or populated until either: +** Otherwise, this instruction checks if cursor P1 contains an entry +** where the first (N-1) fields match but the rowid value at the end +** of the index entry is not R. If there is no such entry, control jumps +** to instruction P2. Otherwise, the rowid of the conflicting index +** entry is copied to register P3 and control falls through to the next +** instruction. ** -** 1) The in-memory representation grows too large for the allocated -** buffer, or -** 2) The sqlite3JournalCreate() function is called. +** See also: NotFound, NotExists, Found */ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +case OP_IsUnique: { /* jump, in3 */ +#if 0 /* local variables moved into u.bc */ + u16 ii; + VdbeCursor *pCx; + BtCursor *pCrsr; + u16 nField; + Mem *aMx; + UnpackedRecord r; /* B-Tree index search key */ + i64 R; /* Rowid stored in register P3 */ +#endif /* local variables moved into u.bc */ + pIn3 = &aMem[pOp->p3]; + u.bc.aMx = &aMem[pOp->p4.i]; + /* Assert that the values of parameters P1 and P4 are in range. */ + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem ); + assert( pOp->p1>=0 && pOp->p1nCursor ); -/* -** A JournalFile object is a subclass of sqlite3_file used by -** as an open file handle for journal files. -*/ -struct JournalFile { - sqlite3_io_methods *pMethod; /* I/O methods on journal files */ - int nBuf; /* Size of zBuf[] in bytes */ - char *zBuf; /* Space to buffer journal writes */ - int iSize; /* Amount of zBuf[] currently used */ - int flags; /* xOpen flags */ - sqlite3_vfs *pVfs; /* The "real" underlying VFS */ - sqlite3_file *pReal; /* The "real" underlying file descriptor */ - const char *zJournal; /* Name of the journal file */ -}; -typedef struct JournalFile JournalFile; + /* Find the index cursor. */ + u.bc.pCx = p->apCsr[pOp->p1]; + assert( u.bc.pCx->deferredMoveto==0 ); + u.bc.pCx->seekResult = 0; + u.bc.pCx->cacheStatus = CACHE_STALE; + u.bc.pCrsr = u.bc.pCx->pCursor; -/* -** If it does not already exists, create and populate the on-disk file -** for JournalFile p. -*/ -static int createFile(JournalFile *p){ - int rc = SQLITE_OK; - if( !p->pReal ){ - sqlite3_file *pReal = (sqlite3_file *)&p[1]; - rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); - if( rc==SQLITE_OK ){ - p->pReal = pReal; - if( p->iSize>0 ){ - assert(p->iSize<=p->nBuf); - rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); - } + /* If any of the values are NULL, take the jump. */ + u.bc.nField = u.bc.pCx->pKeyInfo->nField; + for(u.bc.ii=0; u.bc.iip2 - 1; + u.bc.pCrsr = 0; + break; } } - return rc; -} + assert( (u.bc.aMx[u.bc.nField].flags & MEM_Null)==0 ); -/* -** Close the file. -*/ -static int jrnlClose(sqlite3_file *pJfd){ - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - sqlite3OsClose(p->pReal); - } - sqlite3_free(p->zBuf); - return SQLITE_OK; -} + if( u.bc.pCrsr!=0 ){ + /* Populate the index search key. */ + u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo; + u.bc.r.nField = u.bc.nField + 1; + u.bc.r.flags = UNPACKED_PREFIX_SEARCH; + u.bc.r.aMem = u.bc.aMx; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ipReal ){ - rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); - }else if( (iAmt+iOfst)>p->iSize ){ - rc = SQLITE_IOERR_SHORT_READ; - }else{ - memcpy(zBuf, &p->zBuf[iOfst], iAmt); - } - return rc; -} + /* Extract the value of u.bc.R from register P3. */ + sqlite3VdbeMemIntegerify(pIn3); + u.bc.R = pIn3->u.i; -/* -** Write data to the file. -*/ -static int jrnlWrite( - sqlite3_file *pJfd, /* The journal file into which to write */ - const void *zBuf, /* Take data to be written from here */ - int iAmt, /* Number of bytes to write */ - sqlite_int64 iOfst /* Begin writing at this offset into the file */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ - rc = createFile(p); - } - if( rc==SQLITE_OK ){ - if( p->pReal ){ - rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); + /* Search the B-Tree index. If no conflicting record is found, jump + ** to P2. Otherwise, copy the rowid of the conflicting record to + ** register P3 and fall through to the next instruction. */ + rc = sqlite3BtreeMovetoUnpacked(u.bc.pCrsr, &u.bc.r, 0, 0, &u.bc.pCx->seekResult); + if( (u.bc.r.flags & UNPACKED_PREFIX_SEARCH) || u.bc.r.rowid==u.bc.R ){ + pc = pOp->p2 - 1; }else{ - memcpy(&p->zBuf[iOfst], zBuf, iAmt); - if( p->iSize<(iOfst+iAmt) ){ - p->iSize = (iOfst+iAmt); - } + pIn3->u.i = u.bc.r.rowid; } } - return rc; -} - -/* -** Truncate the file. -*/ -static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsTruncate(p->pReal, size); - }else if( sizeiSize ){ - p->iSize = size; - } - return rc; -} - -/* -** Sync the file. -*/ -static int jrnlSync(sqlite3_file *pJfd, int flags){ - int rc; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsSync(p->pReal, flags); - }else{ - rc = SQLITE_OK; - } - return rc; -} - -/* -** Query the size of the file in bytes. -*/ -static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsFileSize(p->pReal, pSize); - }else{ - *pSize = (sqlite_int64) p->iSize; - } - return rc; + break; } -/* -** Table of methods for JournalFile sqlite3_file object. +/* Opcode: NotExists P1 P2 P3 * * +** +** Use the content of register P3 as an integer key. If a record +** with that key does not exist in table of P1, then jump to P2. +** If the record does exist, then fall through. The cursor is left +** pointing to the record if it exists. +** +** The difference between this operation and NotFound is that this +** operation assumes the key is an integer and that P1 is a table whereas +** NotFound assumes key is a blob constructed from MakeRecord and +** P1 is an index. +** +** See also: Found, NotFound, IsUnique */ -static struct sqlite3_io_methods JournalFileMethods = { - 1, /* iVersion */ - jrnlClose, /* xClose */ - jrnlRead, /* xRead */ - jrnlWrite, /* xWrite */ - jrnlTruncate, /* xTruncate */ - jrnlSync, /* xSync */ - jrnlFileSize, /* xFileSize */ - 0, /* xLock */ - 0, /* xUnlock */ - 0, /* xCheckReservedLock */ - 0, /* xFileControl */ - 0, /* xSectorSize */ - 0, /* xDeviceCharacteristics */ - 0, /* xShmMap */ - 0, /* xShmLock */ - 0, /* xShmBarrier */ - 0 /* xShmUnmap */ -}; +case OP_NotExists: { /* jump, in3 */ +#if 0 /* local variables moved into u.bd */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + u64 iKey; +#endif /* local variables moved into u.bd */ -/* -** Open a journal file. -*/ -SQLITE_PRIVATE int sqlite3JournalOpen( - sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ - const char *zName, /* Name of the journal file */ - sqlite3_file *pJfd, /* Preallocated, blank file handle */ - int flags, /* Opening flags */ - int nBuf /* Bytes buffered before opening the file */ -){ - JournalFile *p = (JournalFile *)pJfd; - memset(p, 0, sqlite3JournalSize(pVfs)); - if( nBuf>0 ){ - p->zBuf = sqlite3MallocZero(nBuf); - if( !p->zBuf ){ - return SQLITE_NOMEM; + pIn3 = &aMem[pOp->p3]; + assert( pIn3->flags & MEM_Int ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bd.pC = p->apCsr[pOp->p1]; + assert( u.bd.pC!=0 ); + assert( u.bd.pC->isTable ); + assert( u.bd.pC->pseudoTableReg==0 ); + u.bd.pCrsr = u.bd.pC->pCursor; + if( u.bd.pCrsr!=0 ){ + u.bd.res = 0; + u.bd.iKey = pIn3->u.i; + rc = sqlite3BtreeMovetoUnpacked(u.bd.pCrsr, 0, u.bd.iKey, 0, &u.bd.res); + u.bd.pC->lastRowid = pIn3->u.i; + u.bd.pC->rowidIsValid = u.bd.res==0 ?1:0; + u.bd.pC->nullRow = 0; + u.bd.pC->cacheStatus = CACHE_STALE; + u.bd.pC->deferredMoveto = 0; + if( u.bd.res!=0 ){ + pc = pOp->p2 - 1; + assert( u.bd.pC->rowidIsValid==0 ); } + u.bd.pC->seekResult = u.bd.res; }else{ - return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + /* This happens when an attempt to open a read cursor on the + ** sqlite_master table returns SQLITE_EMPTY. + */ + pc = pOp->p2 - 1; + assert( u.bd.pC->rowidIsValid==0 ); + u.bd.pC->seekResult = 0; } - p->pMethod = &JournalFileMethods; - p->nBuf = nBuf; - p->flags = flags; - p->zJournal = zName; - p->pVfs = pVfs; - return SQLITE_OK; + break; } -/* -** If the argument p points to a JournalFile structure, and the underlying -** file has not yet been created, create it now. +/* Opcode: Sequence P1 P2 * * * +** +** Find the next available sequence number for cursor P1. +** Write the sequence number into register P2. +** The sequence number on the cursor is incremented after this +** instruction. */ -SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ - if( p->pMethods!=&JournalFileMethods ){ - return SQLITE_OK; - } - return createFile((JournalFile *)p); +case OP_Sequence: { /* out2-prerelease */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( p->apCsr[pOp->p1]!=0 ); + pOut->u.i = p->apCsr[pOp->p1]->seqCount++; + break; } -/* -** Return the number of bytes required to store a JournalFile that uses vfs -** pVfs to create the underlying on-disk files. -*/ -SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ - return (pVfs->szOsFile+sizeof(JournalFile)); -} -#endif -/************** End of journal.c *********************************************/ -/************** Begin file memjournal.c **************************************/ -/* -** 2008 October 7 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +/* Opcode: NewRowid P1 P2 P3 * * ** -************************************************************************* +** Get a new integer record number (a.k.a "rowid") used as the key to a table. +** The record number is not previously used as a key in the database +** table that cursor P1 points to. The new record number is written +** written to register P2. ** -** This file contains code use to implement an in-memory rollback journal. -** The in-memory rollback journal is used to journal transactions for -** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +** If P3>0 then P3 is a register in the root frame of this VDBE that holds +** the largest previously generated record number. No new record numbers are +** allowed to be less than this value. When this value reaches its maximum, +** an SQLITE_FULL error is generated. The P3 register is updated with the ' +** generated record number. This P3 mechanism is used to help implement the +** AUTOINCREMENT feature. */ +case OP_NewRowid: { /* out2-prerelease */ +#if 0 /* local variables moved into u.be */ + i64 v; /* The new rowid */ + VdbeCursor *pC; /* Cursor of table to get the new rowid */ + int res; /* Result of an sqlite3BtreeLast() */ + int cnt; /* Counter to limit the number of searches */ + Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ + VdbeFrame *pFrame; /* Root frame of VDBE */ +#endif /* local variables moved into u.be */ -/* Forward references to internal structures */ -typedef struct MemJournal MemJournal; -typedef struct FilePoint FilePoint; -typedef struct FileChunk FileChunk; - -/* Space to hold the rollback journal is allocated in increments of -** this many bytes. -** -** The size chosen is a little less than a power of two. That way, -** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This mimimizes wasted space in power-of-two -** memory allocators. -*/ -#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) + u.be.v = 0; + u.be.res = 0; + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.be.pC = p->apCsr[pOp->p1]; + assert( u.be.pC!=0 ); + if( NEVER(u.be.pC->pCursor==0) ){ + /* The zero initialization above is all that is needed */ + }else{ + /* The next rowid or record number (different terms for the same + ** thing) is obtained in a two-step algorithm. + ** + ** First we attempt to find the largest existing rowid and add one + ** to that. But if the largest existing rowid is already the maximum + ** positive integer, we have to fall through to the second + ** probabilistic algorithm + ** + ** The second algorithm is to select a rowid at random and see if + ** it already exists in the table. If it does not exist, we have + ** succeeded. If the random rowid does exist, we select a new one + ** and try again, up to 100 times. + */ + assert( u.be.pC->isTable ); -/* Macro to find the minimum of two numeric values. -*/ -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) +#ifdef SQLITE_32BIT_ROWID +# define MAX_ROWID 0x7fffffff +#else + /* Some compilers complain about constants of the form 0x7fffffffffffffff. + ** Others complain about 0x7ffffffffffffffffLL. The following macro seems + ** to provide the constant while making all compilers happy. + */ +# define MAX_ROWID (i64)( (((u64)0x7fffffff)<<32) | (u64)0xffffffff ) #endif -/* -** The rollback journal is composed of a linked list of these structures. -*/ -struct FileChunk { - FileChunk *pNext; /* Next chunk in the journal */ - u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ -}; - -/* -** An instance of this object serves as a cursor into the rollback journal. -** The cursor can be either for reading or writing. -*/ -struct FilePoint { - sqlite3_int64 iOffset; /* Offset from the beginning of the file */ - FileChunk *pChunk; /* Specific chunk into which cursor points */ -}; - -/* -** This subclass is a subclass of sqlite3_file. Each open memory-journal -** is an instance of this class. -*/ -struct MemJournal { - sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ - FileChunk *pFirst; /* Head of in-memory chunk-list */ - FilePoint endpoint; /* Pointer to the end of the file */ - FilePoint readpoint; /* Pointer to the end of the last xRead() */ -}; + if( !u.be.pC->useRandomRowid ){ + u.be.v = sqlite3BtreeGetCachedRowid(u.be.pC->pCursor); + if( u.be.v==0 ){ + rc = sqlite3BtreeLast(u.be.pC->pCursor, &u.be.res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( u.be.res ){ + u.be.v = 1; /* IMP: R-61914-48074 */ + }else{ + assert( sqlite3BtreeCursorIsValid(u.be.pC->pCursor) ); + rc = sqlite3BtreeKeySize(u.be.pC->pCursor, &u.be.v); + assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ + if( u.be.v==MAX_ROWID ){ + u.be.pC->useRandomRowid = 1; + }else{ + u.be.v++; /* IMP: R-29538-34987 */ + } + } + } -/* -** Read data from the in-memory journal file. This is the implementation -** of the sqlite3_vfs.xRead method. -*/ -static int memjrnlRead( - sqlite3_file *pJfd, /* The journal file from which to read */ - void *zBuf, /* Put the results here */ - int iAmt, /* Number of bytes to read */ - sqlite_int64 iOfst /* Begin reading at this offset */ -){ - MemJournal *p = (MemJournal *)pJfd; - u8 *zOut = zBuf; - int nRead = iAmt; - int iChunkOffset; - FileChunk *pChunk; +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( pOp->p3 ){ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3>0 ); + if( p->pFrame ){ + for(u.be.pFrame=p->pFrame; u.be.pFrame->pParent; u.be.pFrame=u.be.pFrame->pParent); + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=u.be.pFrame->nMem ); + u.be.pMem = &u.be.pFrame->aMem[pOp->p3]; + }else{ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=p->nMem ); + u.be.pMem = &aMem[pOp->p3]; + memAboutToChange(p, u.be.pMem); + } + assert( memIsValid(u.be.pMem) ); - /* SQLite never tries to read past the end of a rollback journal file */ - assert( iOfst+iAmt<=p->endpoint.iOffset ); + REGISTER_TRACE(pOp->p3, u.be.pMem); + sqlite3VdbeMemIntegerify(u.be.pMem); + assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ + if( u.be.pMem->u.i==MAX_ROWID || u.be.pC->useRandomRowid ){ + rc = SQLITE_FULL; /* IMP: R-12275-61338 */ + goto abort_due_to_error; + } + if( u.be.vu.i+1 ){ + u.be.v = u.be.pMem->u.i + 1; + } + u.be.pMem->u.i = u.be.v; + } +#endif - if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ - sqlite3_int64 iOff = 0; - for(pChunk=p->pFirst; - ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; - pChunk=pChunk->pNext - ){ - iOff += JOURNAL_CHUNKSIZE; + sqlite3BtreeSetCachedRowid(u.be.pC->pCursor, u.be.vreadpoint.pChunk; + if( u.be.pC->useRandomRowid ){ + /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the + ** largest possible integer (9223372036854775807) then the database + ** engine starts picking positive candidate ROWIDs at random until + ** it finds one that is not previously used. */ + assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is + ** an AUTOINCREMENT table. */ + /* on the first attempt, simply do one more than previous */ + u.be.v = lastRowid; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + u.be.v++; /* ensure non-zero */ + u.be.cnt = 0; + while( ((rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, + 0, &u.be.res))==SQLITE_OK) + && (u.be.res==0) + && (++u.be.cnt<100)){ + /* collision - try another random rowid */ + sqlite3_randomness(sizeof(u.be.v), &u.be.v); + if( u.be.cnt<5 ){ + /* try "small" random rowids for the initial attempts */ + u.be.v &= 0xffffff; + }else{ + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + } + u.be.v++; /* ensure non-zero */ + } + if( rc==SQLITE_OK && u.be.res==0 ){ + rc = SQLITE_FULL; /* IMP: R-38219-53002 */ + goto abort_due_to_error; + } + assert( u.be.v>0 ); /* EV: R-40812-03570 */ + } + u.be.pC->rowidIsValid = 0; + u.be.pC->deferredMoveto = 0; + u.be.pC->cacheStatus = CACHE_STALE; } - - iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); - do { - int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; - int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); - memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); - zOut += nCopy; - nRead -= iSpace; - iChunkOffset = 0; - } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = iOfst+iAmt; - p->readpoint.pChunk = pChunk; - - return SQLITE_OK; + pOut->u.i = u.be.v; + break; } -/* -** Write data to the file. +/* Opcode: Insert P1 P2 P3 P4 P5 +** +** Write an entry into the table of cursor P1. A new entry is +** created if it doesn't already exist or the data for an existing +** entry is overwritten. The data is the value MEM_Blob stored in register +** number P2. The key is stored in register P3. The key must +** be a MEM_Int. +** +** If the OPFLAG_NCHANGE flag of P5 is set, then the row change count is +** incremented (otherwise not). If the OPFLAG_LASTROWID flag of P5 is set, +** then rowid is stored for subsequent return by the +** sqlite3_last_insert_rowid() function (otherwise it is unmodified). +** +** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of +** the last seek operation (OP_NotExists) was a success, then this +** operation will not attempt to find the appropriate row before doing +** the insert but will instead overwrite the row that the cursor is +** currently pointing to. Presumably, the prior OP_NotExists opcode +** has already positioned the cursor correctly. This is an optimization +** that boosts performance by avoiding redundant seeks. +** +** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an +** UPDATE operation. Otherwise (if the flag is clear) then this opcode +** is part of an INSERT operation. The difference is only important to +** the update hook. +** +** Parameter P4 may point to a string containing the table-name, or +** may be NULL. If it is not NULL, then the update-hook +** (sqlite3.xUpdateCallback) is invoked following a successful insert. +** +** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically +** allocated, then ownership of P2 is transferred to the pseudo-cursor +** and register P2 becomes ephemeral. If the cursor is changed, the +** value of register P2 will then change. Make sure this does not +** cause any problems.) +** +** This instruction only works on tables. The equivalent instruction +** for indices is OP_IdxInsert. */ -static int memjrnlWrite( - sqlite3_file *pJfd, /* The journal file into which to write */ - const void *zBuf, /* Take data to be written from here */ - int iAmt, /* Number of bytes to write */ - sqlite_int64 iOfst /* Begin writing at this offset into the file */ -){ - MemJournal *p = (MemJournal *)pJfd; - int nWrite = iAmt; - u8 *zWrite = (u8 *)zBuf; - - /* An in-memory journal file should only ever be appended to. Random - ** access writes are not required by sqlite. - */ - assert( iOfst==p->endpoint.iOffset ); - UNUSED_PARAMETER(iOfst); - - while( nWrite>0 ){ - FileChunk *pChunk = p->endpoint.pChunk; - int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); - int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); +/* Opcode: InsertInt P1 P2 P3 P4 P5 +** +** This works exactly like OP_Insert except that the key is the +** integer value P3, not the value of the integer stored in register P3. +*/ +case OP_Insert: +case OP_InsertInt: { +#if 0 /* local variables moved into u.bf */ + Mem *pData; /* MEM cell holding data for the record to be inserted */ + Mem *pKey; /* MEM cell holding key for the record */ + i64 iKey; /* The integer ROWID or key for the record to be inserted */ + VdbeCursor *pC; /* Cursor to table into which insert is written */ + int nZero; /* Number of zero-bytes to append */ + int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ + const char *zDb; /* database name - used by the update hook */ + const char *zTbl; /* Table name - used by the opdate hook */ + int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ +#endif /* local variables moved into u.bf */ - if( iChunkOffset==0 ){ - /* New chunk is required to extend the file. */ - FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); - if( !pNew ){ - return SQLITE_IOERR_NOMEM; - } - pNew->pNext = 0; - if( pChunk ){ - assert( p->pFirst ); - pChunk->pNext = pNew; - }else{ - assert( !p->pFirst ); - p->pFirst = pNew; - } - p->endpoint.pChunk = pNew; - } + u.bf.pData = &aMem[pOp->p2]; + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( memIsValid(u.bf.pData) ); + u.bf.pC = p->apCsr[pOp->p1]; + assert( u.bf.pC!=0 ); + assert( u.bf.pC->pCursor!=0 ); + assert( u.bf.pC->pseudoTableReg==0 ); + assert( u.bf.pC->isTable ); + REGISTER_TRACE(pOp->p2, u.bf.pData); - memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); - zWrite += iSpace; - nWrite -= iSpace; - p->endpoint.iOffset += iSpace; + if( pOp->opcode==OP_Insert ){ + u.bf.pKey = &aMem[pOp->p3]; + assert( u.bf.pKey->flags & MEM_Int ); + assert( memIsValid(u.bf.pKey) ); + REGISTER_TRACE(pOp->p3, u.bf.pKey); + u.bf.iKey = u.bf.pKey->u.i; + }else{ + assert( pOp->opcode==OP_InsertInt ); + u.bf.iKey = pOp->p3; } - return SQLITE_OK; -} - -/* -** Truncate the file. -*/ -static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ - MemJournal *p = (MemJournal *)pJfd; - FileChunk *pChunk; - assert(size==0); - UNUSED_PARAMETER(size); - pChunk = p->pFirst; - while( pChunk ){ - FileChunk *pTmp = pChunk; - pChunk = pChunk->pNext; - sqlite3_free(pTmp); + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = u.bf.iKey; + if( u.bf.pData->flags & MEM_Null ){ + u.bf.pData->z = 0; + u.bf.pData->n = 0; + }else{ + assert( u.bf.pData->flags & (MEM_Blob|MEM_Str) ); } - sqlite3MemJournalOpen(pJfd); - return SQLITE_OK; -} + u.bf.seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? u.bf.pC->seekResult : 0); + if( u.bf.pData->flags & MEM_Zero ){ + u.bf.nZero = u.bf.pData->u.nZero; + }else{ + u.bf.nZero = 0; + } + sqlite3BtreeSetCachedRowid(u.bf.pC->pCursor, 0); + rc = sqlite3BtreeInsert(u.bf.pC->pCursor, 0, u.bf.iKey, + u.bf.pData->z, u.bf.pData->n, u.bf.nZero, + pOp->p5 & OPFLAG_APPEND, u.bf.seekResult + ); + u.bf.pC->rowidIsValid = 0; + u.bf.pC->deferredMoveto = 0; + u.bf.pC->cacheStatus = CACHE_STALE; -/* -** Close the file. -*/ -static int memjrnlClose(sqlite3_file *pJfd){ - memjrnlTruncate(pJfd, 0); - return SQLITE_OK; + /* Invoke the update-hook if required. */ + if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ + u.bf.zDb = db->aDb[u.bf.pC->iDb].zName; + u.bf.zTbl = pOp->p4.z; + u.bf.op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); + assert( u.bf.pC->isTable ); + db->xUpdateCallback(db->pUpdateArg, u.bf.op, u.bf.zDb, u.bf.zTbl, u.bf.iKey); + assert( u.bf.pC->iDb>=0 ); + } + break; } - -/* -** Sync the file. +/* Opcode: Delete P1 P2 * P4 * ** -** Syncing an in-memory journal is a no-op. And, in fact, this routine -** is never called in a working implementation. This implementation -** exists purely as a contingency, in case some malfunction in some other -** part of SQLite causes Sync to be called by mistake. +** Delete the record at which the P1 cursor is currently pointing. +** +** The cursor will be left pointing at either the next or the previous +** record in the table. If it is left pointing at the next record, then +** the next Next instruction will be a no-op. Hence it is OK to delete +** a record from within an Next loop. +** +** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is +** incremented (otherwise not). +** +** P1 must not be pseudo-table. It has to be a real table with +** multiple rows. +** +** If P4 is not NULL, then it is the name of the table that P1 is +** pointing to. The update hook will be invoked, if it exists. +** If P4 is not NULL then the P1 cursor must have been positioned +** using OP_NotFound prior to invoking this opcode. */ -static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - return SQLITE_OK; -} +case OP_Delete: { +#if 0 /* local variables moved into u.bg */ + i64 iKey; + VdbeCursor *pC; +#endif /* local variables moved into u.bg */ -/* -** Query the size of the file in bytes. -*/ -static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ - MemJournal *p = (MemJournal *)pJfd; - *pSize = (sqlite_int64) p->endpoint.iOffset; - return SQLITE_OK; -} + u.bg.iKey = 0; + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bg.pC = p->apCsr[pOp->p1]; + assert( u.bg.pC!=0 ); + assert( u.bg.pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ -/* -** Table of methods for MemJournal sqlite3_file object. -*/ -static const struct sqlite3_io_methods MemJournalMethods = { - 1, /* iVersion */ - memjrnlClose, /* xClose */ - memjrnlRead, /* xRead */ - memjrnlWrite, /* xWrite */ - memjrnlTruncate, /* xTruncate */ - memjrnlSync, /* xSync */ - memjrnlFileSize, /* xFileSize */ - 0, /* xLock */ - 0, /* xUnlock */ - 0, /* xCheckReservedLock */ - 0, /* xFileControl */ - 0, /* xSectorSize */ - 0, /* xDeviceCharacteristics */ - 0, /* xShmMap */ - 0, /* xShmLock */ - 0, /* xShmBarrier */ - 0 /* xShmUnlock */ -}; + /* If the update-hook will be invoked, set u.bg.iKey to the rowid of the + ** row being deleted. + */ + if( db->xUpdateCallback && pOp->p4.z ){ + assert( u.bg.pC->isTable ); + assert( u.bg.pC->rowidIsValid ); /* lastRowid set by previous OP_NotFound */ + u.bg.iKey = u.bg.pC->lastRowid; + } -/* -** Open a journal file. -*/ -SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ - MemJournal *p = (MemJournal *)pJfd; - assert( EIGHT_BYTE_ALIGNMENT(p) ); - memset(p, 0, sqlite3MemJournalSize()); - p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; -} + /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or + ** OP_Column on the same table without any intervening operations that + ** might move or invalidate the cursor. Hence cursor u.bg.pC is always pointing + ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation + ** below is always a no-op and cannot fail. We will run it anyhow, though, + ** to guard against future changes to the code generator. + **/ + assert( u.bg.pC->deferredMoveto==0 ); + rc = sqlite3VdbeCursorMoveto(u.bg.pC); + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; -/* -** Return true if the file-handle passed as an argument is -** an in-memory journal -*/ -SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ - return pJfd->pMethods==&MemJournalMethods; -} + sqlite3BtreeSetCachedRowid(u.bg.pC->pCursor, 0); + rc = sqlite3BtreeDelete(u.bg.pC->pCursor); + u.bg.pC->cacheStatus = CACHE_STALE; -/* -** Return the number of bytes required to store a MemJournal file descriptor. -*/ -SQLITE_PRIVATE int sqlite3MemJournalSize(void){ - return sizeof(MemJournal); + /* Invoke the update-hook if required. */ + if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){ + const char *zDb = db->aDb[u.bg.pC->iDb].zName; + const char *zTbl = pOp->p4.z; + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, zTbl, u.bg.iKey); + assert( u.bg.pC->iDb>=0 ); + } + if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; + break; } - -/************** End of memjournal.c ******************************************/ -/************** Begin file walker.c ******************************************/ -/* -** 2008 August 16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +/* Opcode: ResetCount * * * * * ** -************************************************************************* -** This file contains routines used for walking the parser tree for -** an SQL statement. +** The value of the change counter is copied to the database handle +** change counter (returned by subsequent calls to sqlite3_changes()). +** Then the VMs internal change counter resets to 0. +** This is used by trigger programs. */ +case OP_ResetCount: { + sqlite3VdbeSetChanges(db, p->nChange); + p->nChange = 0; + break; +} - -/* -** Walk an expression tree. Invoke the callback once for each node -** of the expression, while decending. (In other words, the callback -** is invoked before visiting children.) -** -** The return value from the callback should be one of the WRC_* -** constants to specify how to proceed with the walk. +/* Opcode: RowData P1 P2 * * * ** -** WRC_Continue Continue descending down the tree. +** Write into register P2 the complete row data for cursor P1. +** There is no interpretation of the data. +** It is just copied onto the P2 register exactly as +** it is found in the database file. ** -** WRC_Prune Do not descend into child nodes. But allow -** the walk to continue with sibling nodes. +** If the P1 cursor must be pointing to a valid row (not a NULL row) +** of a real table, not a pseudo-table. +*/ +/* Opcode: RowKey P1 P2 * * * ** -** WRC_Abort Do no more callbacks. Unwind the stack and -** return the top-level walk call. +** Write into register P2 the complete row key for cursor P1. +** There is no interpretation of the data. +** The key is copied onto the P3 register exactly as +** it is found in the database file. ** -** The return value from this routine is WRC_Abort to abandon the tree walk -** and WRC_Continue to continue. +** If the P1 cursor must be pointing to a valid row (not a NULL row) +** of a real table, not a pseudo-table. */ -SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ - int rc; - if( pExpr==0 ) return WRC_Continue; - testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); - testcase( ExprHasProperty(pExpr, EP_Reduced) ); - rc = pWalker->xExprCallback(pWalker, pExpr); - if( rc==WRC_Continue - && !ExprHasAnyProperty(pExpr,EP_TokenOnly) ){ - if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else{ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; - } - } - return rc & WRC_Abort; -} +case OP_RowKey: +case OP_RowData: { +#if 0 /* local variables moved into u.bh */ + VdbeCursor *pC; + BtCursor *pCrsr; + u32 n; + i64 n64; +#endif /* local variables moved into u.bh */ -/* -** Call sqlite3WalkExpr() for every expression in list p or until -** an abort request is seen. -*/ -SQLITE_PRIVATE int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ - int i; - struct ExprList_item *pItem; - if( p ){ - for(i=p->nExpr, pItem=p->a; i>0; i--, pItem++){ - if( sqlite3WalkExpr(pWalker, pItem->pExpr) ) return WRC_Abort; - } - } - return WRC_Continue; -} + pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); -/* -** Walk all expressions associated with SELECT statement p. Do -** not invoke the SELECT callback on p, but do (of course) invoke -** any expr callbacks and SELECT callbacks that come from subqueries. -** Return WRC_Abort or WRC_Continue. -*/ -SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ - if( sqlite3WalkExprList(pWalker, p->pEList) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, p->pWhere) ) return WRC_Abort; - if( sqlite3WalkExprList(pWalker, p->pGroupBy) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; - if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, p->pOffset) ) return WRC_Abort; - return WRC_Continue; + /* Note that RowKey and RowData are really exactly the same instruction */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bh.pC = p->apCsr[pOp->p1]; + assert( u.bh.pC->isTable || pOp->opcode==OP_RowKey ); + assert( u.bh.pC->isIndex || pOp->opcode==OP_RowData ); + assert( u.bh.pC!=0 ); + assert( u.bh.pC->nullRow==0 ); + assert( u.bh.pC->pseudoTableReg==0 ); + assert( u.bh.pC->pCursor!=0 ); + u.bh.pCrsr = u.bh.pC->pCursor; + assert( sqlite3BtreeCursorIsValid(u.bh.pCrsr) ); + + /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or + ** OP_Rewind/Op_Next with no intervening instructions that might invalidate + ** the cursor. Hence the following sqlite3VdbeCursorMoveto() call is always + ** a no-op and can never fail. But we leave it in place as a safety. + */ + assert( u.bh.pC->deferredMoveto==0 ); + rc = sqlite3VdbeCursorMoveto(u.bh.pC); + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + + if( u.bh.pC->isIndex ){ + assert( !u.bh.pC->isTable ); + rc = sqlite3BtreeKeySize(u.bh.pCrsr, &u.bh.n64); + assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ + if( u.bh.n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + u.bh.n = (u32)u.bh.n64; + }else{ + rc = sqlite3BtreeDataSize(u.bh.pCrsr, &u.bh.n); + assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ + if( u.bh.n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + } + if( sqlite3VdbeMemGrow(pOut, u.bh.n, 0) ){ + goto no_mem; + } + pOut->n = u.bh.n; + MemSetTypeFlag(pOut, MEM_Blob); + if( u.bh.pC->isIndex ){ + rc = sqlite3BtreeKey(u.bh.pCrsr, 0, u.bh.n, pOut->z); + }else{ + rc = sqlite3BtreeData(u.bh.pCrsr, 0, u.bh.n, pOut->z); + } + pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ + UPDATE_MAX_BLOBSIZE(pOut); + break; } -/* -** Walk the parse trees associated with all subqueries in the -** FROM clause of SELECT statement p. Do not invoke the select -** callback on p, but do invoke it on each FROM clause subquery -** and on any subqueries further down in the tree. Return -** WRC_Abort or WRC_Continue; +/* Opcode: Rowid P1 P2 * * * +** +** Store in register P2 an integer which is the key of the table entry that +** P1 is currently point to. +** +** P1 can be either an ordinary table or a virtual table. There used to +** be a separate OP_VRowid opcode for use with virtual tables, but this +** one opcode now works for both table types. */ -SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ - SrcList *pSrc; - int i; - struct SrcList_item *pItem; +case OP_Rowid: { /* out2-prerelease */ +#if 0 /* local variables moved into u.bi */ + VdbeCursor *pC; + i64 v; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; +#endif /* local variables moved into u.bi */ - pSrc = p->pSrc; - if( ALWAYS(pSrc) ){ - for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ - return WRC_Abort; - } + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bi.pC = p->apCsr[pOp->p1]; + assert( u.bi.pC!=0 ); + assert( u.bi.pC->pseudoTableReg==0 ); + if( u.bi.pC->nullRow ){ + pOut->flags = MEM_Null; + break; + }else if( u.bi.pC->deferredMoveto ){ + u.bi.v = u.bi.pC->movetoTarget; +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( u.bi.pC->pVtabCursor ){ + u.bi.pVtab = u.bi.pC->pVtabCursor->pVtab; + u.bi.pModule = u.bi.pVtab->pModule; + assert( u.bi.pModule->xRowid ); + rc = u.bi.pModule->xRowid(u.bi.pC->pVtabCursor, &u.bi.v); + importVtabErrMsg(p, u.bi.pVtab); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + }else{ + assert( u.bi.pC->pCursor!=0 ); + rc = sqlite3VdbeCursorMoveto(u.bi.pC); + if( rc ) goto abort_due_to_error; + if( u.bi.pC->rowidIsValid ){ + u.bi.v = u.bi.pC->lastRowid; + }else{ + rc = sqlite3BtreeKeySize(u.bi.pC->pCursor, &u.bi.v); + assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ } } - return WRC_Continue; -} + pOut->u.i = u.bi.v; + break; +} -/* -** Call sqlite3WalkExpr() for every expression in Select statement p. -** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. -** -** Return WRC_Continue under normal conditions. Return WRC_Abort if -** there is an abort request. +/* Opcode: NullRow P1 * * * * ** -** If the Walker does not have an xSelectCallback() then this routine -** is a no-op returning WRC_Continue. +** Move the cursor P1 to a null row. Any OP_Column operations +** that occur while the cursor is on the null row will always +** write a NULL. */ -SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ - int rc; - if( p==0 || pWalker->xSelectCallback==0 ) return WRC_Continue; - rc = WRC_Continue; - while( p ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; - if( sqlite3WalkSelectExpr(pWalker, p) ) return WRC_Abort; - if( sqlite3WalkSelectFrom(pWalker, p) ) return WRC_Abort; - p = p->pPrior; +case OP_NullRow: { +#if 0 /* local variables moved into u.bj */ + VdbeCursor *pC; +#endif /* local variables moved into u.bj */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bj.pC = p->apCsr[pOp->p1]; + assert( u.bj.pC!=0 ); + u.bj.pC->nullRow = 1; + u.bj.pC->rowidIsValid = 0; + if( u.bj.pC->pCursor ){ + sqlite3BtreeClearCursor(u.bj.pC->pCursor); } - return rc & WRC_Abort; + break; } -/************** End of walker.c **********************************************/ -/************** Begin file resolve.c *****************************************/ -/* -** 2008 August 18 +/* Opcode: Last P1 P2 * * * ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** The next use of the Rowid or Column or Next instruction for P1 +** will refer to the last entry in the database table or index. +** If the table or index is empty and P2>0, then jump immediately to P2. +** If P2 is 0 or if the table or index is not empty, fall through +** to the following instruction. +*/ +case OP_Last: { /* jump */ +#if 0 /* local variables moved into u.bk */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; +#endif /* local variables moved into u.bk */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bk.pC = p->apCsr[pOp->p1]; + assert( u.bk.pC!=0 ); + u.bk.pCrsr = u.bk.pC->pCursor; + if( u.bk.pCrsr==0 ){ + u.bk.res = 1; + }else{ + rc = sqlite3BtreeLast(u.bk.pCrsr, &u.bk.res); + } + u.bk.pC->nullRow = (u8)u.bk.res; + u.bk.pC->deferredMoveto = 0; + u.bk.pC->rowidIsValid = 0; + u.bk.pC->cacheStatus = CACHE_STALE; + if( pOp->p2>0 && u.bk.res ){ + pc = pOp->p2 - 1; + } + break; +} + + +/* Opcode: Sort P1 P2 * * * ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This opcode does exactly the same thing as OP_Rewind except that +** it increments an undocumented global variable used for testing. ** -************************************************************************* +** Sorting is accomplished by writing records into a sorting index, +** then rewinding that index and playing it back from beginning to +** end. We use the OP_Sort opcode instead of OP_Rewind to do the +** rewinding so that the global variable will be incremented and +** regression tests can determine whether or not the optimizer is +** correctly optimizing out sorts. +*/ +case OP_Sort: { /* jump */ +#ifdef SQLITE_TEST + sqlite3_sort_count++; + sqlite3_search_count--; +#endif + p->aCounter[SQLITE_STMTSTATUS_SORT-1]++; + /* Fall through into OP_Rewind */ +} +/* Opcode: Rewind P1 P2 * * * ** -** This file contains routines used for walking the parser tree and -** resolve all identifiers by associating them with a particular -** table and column. +** The next use of the Rowid or Column or Next instruction for P1 +** will refer to the first entry in the database table or index. +** If the table or index is empty and P2>0, then jump immediately to P2. +** If P2 is 0 or if the table or index is not empty, fall through +** to the following instruction. */ +case OP_Rewind: { /* jump */ +#if 0 /* local variables moved into u.bl */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; +#endif /* local variables moved into u.bl */ -/* -** Turn the pExpr expression into an alias for the iCol-th column of the -** result set in pEList. + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bl.pC = p->apCsr[pOp->p1]; + assert( u.bl.pC!=0 ); + u.bl.res = 1; + if( (u.bl.pCrsr = u.bl.pC->pCursor)!=0 ){ + rc = sqlite3BtreeFirst(u.bl.pCrsr, &u.bl.res); + u.bl.pC->atFirst = u.bl.res==0 ?1:0; + u.bl.pC->deferredMoveto = 0; + u.bl.pC->cacheStatus = CACHE_STALE; + u.bl.pC->rowidIsValid = 0; + } + u.bl.pC->nullRow = (u8)u.bl.res; + assert( pOp->p2>0 && pOp->p2nOp ); + if( u.bl.res ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: Next P1 P2 * * P5 ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. +** Advance cursor P1 so that it points to the next key/data pair in its +** table or index. If there are no more key/value pairs then fall through +** to the following instruction. But if the cursor advance was successful, +** jump immediately to P2. ** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. +** The P1 cursor must be for a real table, not a pseudo-table. ** -** Hack: The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: +** If P5 is positive and the jump is taken, then event counter +** number P5-1 in the prepared statement is incremented. ** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x +** See also: Prev +*/ +/* Opcode: Prev P1 P2 * * P5 ** -** Is equivalent to: +** Back up cursor P1 so that it points to the previous key/data pair in its +** table or index. If there is no previous key/value pairs then fall through +** to the following instruction. But if the cursor backup was successful, +** jump immediately to P2. ** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 +** The P1 cursor must be for a real table, not a pseudo-table. ** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. We might fix this someday. Or -** then again, we might not... +** If P5 is positive and the jump is taken, then event counter +** number P5-1 in the prepared statement is incremented. */ -static void resolveAlias( - Parse *pParse, /* Parsing context */ - ExprList *pEList, /* A result set */ - int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ - Expr *pExpr, /* Transform this into an alias to the result set */ - const char *zType /* "GROUP" or "ORDER" or "" */ -){ - Expr *pOrig; /* The iCol-th column of the result set */ - Expr *pDup; /* Copy of pOrig */ - sqlite3 *db; /* The database connection */ +case OP_Prev: /* jump */ +case OP_Next: { /* jump */ +#if 0 /* local variables moved into u.bm */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; +#endif /* local variables moved into u.bm */ - assert( iCol>=0 && iColnExpr ); - pOrig = pEList->a[iCol].pExpr; - assert( pOrig!=0 ); - assert( pOrig->flags & EP_Resolved ); - db = pParse->db; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - pDup = sqlite3ExprDup(db, pOrig, 0); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - if( pEList->a[iCol].iAlias==0 ){ - pEList->a[iCol].iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].iAlias; - }else if( ExprHasProperty(pOrig, EP_IntValue) || pOrig->u.zToken==0 ){ - pDup = sqlite3ExprDup(db, pOrig, 0); - if( pDup==0 ) return; - }else{ - char *zToken = pOrig->u.zToken; - assert( zToken!=0 ); - pOrig->u.zToken = 0; - pDup = sqlite3ExprDup(db, pOrig, 0); - pOrig->u.zToken = zToken; - if( pDup==0 ) return; - assert( (pDup->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pDup->flags2 |= EP2_MallocedToken; - pDup->u.zToken = sqlite3DbStrDup(db, zToken); + CHECK_FOR_INTERRUPT; + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5<=ArraySize(p->aCounter) ); + u.bm.pC = p->apCsr[pOp->p1]; + if( u.bm.pC==0 ){ + break; /* See ticket #2273 */ } - if( pExpr->flags & EP_ExpCollate ){ - pDup->pColl = pExpr->pColl; - pDup->flags |= EP_ExpCollate; + u.bm.pCrsr = u.bm.pC->pCursor; + if( u.bm.pCrsr==0 ){ + u.bm.pC->nullRow = 1; + break; } - - /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This - ** prevents ExprDelete() from deleting the Expr structure itself, - ** allowing it to be repopulated by the memcpy() on the following line. - */ - ExprSetProperty(pExpr, EP_Static); - sqlite3ExprDelete(db, pExpr); - memcpy(pExpr, pDup, sizeof(*pExpr)); - sqlite3DbFree(db, pDup); + u.bm.res = 1; + assert( u.bm.pC->deferredMoveto==0 ); + rc = pOp->opcode==OP_Next ? sqlite3BtreeNext(u.bm.pCrsr, &u.bm.res) : + sqlite3BtreePrevious(u.bm.pCrsr, &u.bm.res); + u.bm.pC->nullRow = (u8)u.bm.res; + u.bm.pC->cacheStatus = CACHE_STALE; + if( u.bm.res==0 ){ + pc = pOp->p2 - 1; + if( pOp->p5 ) p->aCounter[pOp->p5-1]++; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + } + u.bm.pC->rowidIsValid = 0; + break; } -/* -** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up -** that name in the set of source tables in pSrcList and make the pExpr -** expression node refer back to that source column. The following changes -** are made to pExpr: +/* Opcode: IdxInsert P1 P2 P3 * P5 ** -** pExpr->iDb Set the index in db->aDb[] of the database X -** (even if X is implied). -** pExpr->iTable Set to the cursor number for the table obtained -** from pSrcList. -** pExpr->pTab Points to the Table structure of X.Y (even if -** X and/or Y are implied.) -** pExpr->iColumn Set to the column number within the table. -** pExpr->op Set to TK_COLUMN. -** pExpr->pLeft Any expression this points to is deleted -** pExpr->pRight Any expression this points to is deleted. +** Register P2 holds an SQL index key made using the +** MakeRecord instructions. This opcode writes that key +** into the index P1. Data for the entry is nil. ** -** The zDb variable is the name of the database (the "X"). This value may be -** NULL meaning that name is of the form Y.Z or Z. Any available database -** can be used. The zTable variable is the name of the table (the "Y"). This -** value can be NULL if zDb is also NULL. If zTable is NULL it -** means that the form of the name is Z and that columns from any table -** can be used. +** P3 is a flag that provides a hint to the b-tree layer that this +** insert is likely to be an append. ** -** If the name cannot be resolved unambiguously, leave an error message -** in pParse and return WRC_Abort. Return WRC_Prune on success. +** This instruction only works for indices. The equivalent instruction +** for tables is OP_Insert. */ -static int lookupName( - Parse *pParse, /* The parsing context */ - const char *zDb, /* Name of the database containing table, or NULL */ - const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ - NameContext *pNC, /* The name context used to resolve the name */ - Expr *pExpr /* Make this EXPR node point to the selected column */ -){ - int i, j; /* Loop counters */ - int cnt = 0; /* Number of matching column names */ - int cntTab = 0; /* Number of matching table names */ - sqlite3 *db = pParse->db; /* The database connection */ - struct SrcList_item *pItem; /* Use for looping over pSrcList items */ - struct SrcList_item *pMatch = 0; /* The matching pSrcList item */ - NameContext *pTopNC = pNC; /* First namecontext in the list */ - Schema *pSchema = 0; /* Schema of the expression */ - int isTrigger = 0; +case OP_IdxInsert: { /* in2 */ +#if 0 /* local variables moved into u.bn */ + VdbeCursor *pC; + BtCursor *pCrsr; + int nKey; + const char *zKey; +#endif /* local variables moved into u.bn */ - assert( pNC ); /* the name context cannot be NULL. */ - assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ - assert( ~ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bn.pC = p->apCsr[pOp->p1]; + assert( u.bn.pC!=0 ); + pIn2 = &aMem[pOp->p2]; + assert( pIn2->flags & MEM_Blob ); + u.bn.pCrsr = u.bn.pC->pCursor; + if( ALWAYS(u.bn.pCrsr!=0) ){ + assert( u.bn.pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc==SQLITE_OK ){ + u.bn.nKey = pIn2->n; + u.bn.zKey = pIn2->z; + rc = sqlite3BtreeInsert(u.bn.pCrsr, u.bn.zKey, u.bn.nKey, "", 0, 0, pOp->p3, + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? u.bn.pC->seekResult : 0) + ); + assert( u.bn.pC->deferredMoveto==0 ); + u.bn.pC->cacheStatus = CACHE_STALE; + } + } + break; +} - /* Initialize the node to no-match */ - pExpr->iTable = -1; - pExpr->pTab = 0; - ExprSetIrreducible(pExpr); +/* Opcode: IdxDelete P1 P2 P3 * * +** +** The content of P3 registers starting at register P2 form +** an unpacked index key. This opcode removes that entry from the +** index opened by cursor P1. +*/ +case OP_IdxDelete: { +#if 0 /* local variables moved into u.bo */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + UnpackedRecord r; +#endif /* local variables moved into u.bo */ - /* Start at the inner-most context and move outward until a match is found */ - while( pNC && cnt==0 ){ - ExprList *pEList; - SrcList *pSrcList = pNC->pSrcList; + assert( pOp->p3>0 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=p->nMem+1 ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bo.pC = p->apCsr[pOp->p1]; + assert( u.bo.pC!=0 ); + u.bo.pCrsr = u.bo.pC->pCursor; + if( ALWAYS(u.bo.pCrsr!=0) ){ + u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo; + u.bo.r.nField = (u16)pOp->p3; + u.bo.r.flags = 0; + u.bo.r.aMem = &aMem[pOp->p2]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ideferredMoveto==0 ); + u.bo.pC->cacheStatus = CACHE_STALE; + } + break; +} - if( pSrcList ){ - for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ - Table *pTab; - int iDb; - Column *pCol; - - pTab = pItem->pTab; - assert( pTab!=0 && pTab->zName!=0 ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( pTab->nCol>0 ); - if( zTab ){ - if( pItem->zAlias ){ - char *zTabName = pItem->zAlias; - if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; - }else{ - char *zTabName = pTab->zName; - if( NEVER(zTabName==0) || sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; - } - if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ - continue; - } - } - } - if( 0==(cntTab++) ){ - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; - pSchema = pTab->pSchema; - pMatch = pItem; - } - for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - IdList *pUsing; - cnt++; - pExpr->iTable = pItem->iCursor; - pExpr->pTab = pTab; - pMatch = pItem; - pSchema = pTab->pSchema; - /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ - pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; - if( inSrc-1 ){ - if( pItem[1].jointype & JT_NATURAL ){ - /* If this match occurred in the left table of a natural join, - ** then skip the right table to avoid a duplicate match */ - pItem++; - i++; - }else if( (pUsing = pItem[1].pUsing)!=0 ){ - /* If this match occurs on a column that is in the USING clause - ** of a join, skip the search of the right table of the join - ** to avoid a duplicate match there. */ - int k; - for(k=0; knId; k++){ - if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ){ - pItem++; - i++; - break; - } - } - } - } - break; - } - } - } - } - -#ifndef SQLITE_OMIT_TRIGGER - /* If we have not already resolved the name, then maybe - ** it is a new.* or old.* trigger argument reference - */ - if( zDb==0 && zTab!=0 && cnt==0 && pParse->pTriggerTab!=0 ){ - int op = pParse->eTriggerOp; - Table *pTab = 0; - assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); - if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){ - pExpr->iTable = 1; - pTab = pParse->pTriggerTab; - }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){ - pExpr->iTable = 0; - pTab = pParse->pTriggerTab; - } +/* Opcode: IdxRowid P1 P2 * * * +** +** Write into register P2 an integer which is the last entry in the record at +** the end of the index key pointed to by cursor P1. This integer should be +** the rowid of the table entry to which this index entry points. +** +** See also: Rowid, MakeRecord. +*/ +case OP_IdxRowid: { /* out2-prerelease */ +#if 0 /* local variables moved into u.bp */ + BtCursor *pCrsr; + VdbeCursor *pC; + i64 rowid; +#endif /* local variables moved into u.bp */ - if( pTab ){ - int iCol; - pSchema = pTab->pSchema; - cntTab++; - for(iCol=0; iColnCol; iCol++){ - Column *pCol = &pTab->aCol[iCol]; - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - if( iCol==pTab->iPKey ){ - iCol = -1; - } - break; - } - } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) ){ - iCol = -1; /* IMP: R-44911-55124 */ - } - if( iColnCol ){ - cnt++; - if( iCol<0 ){ - pExpr->affinity = SQLITE_AFF_INTEGER; - }else if( pExpr->iTable==0 ){ - testcase( iCol==31 ); - testcase( iCol==32 ); - pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<iColumn = (i16)iCol; - pExpr->pTab = pTab; - isTrigger = 1; - } + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bp.pC = p->apCsr[pOp->p1]; + assert( u.bp.pC!=0 ); + u.bp.pCrsr = u.bp.pC->pCursor; + pOut->flags = MEM_Null; + if( ALWAYS(u.bp.pCrsr!=0) ){ + rc = sqlite3VdbeCursorMoveto(u.bp.pC); + if( NEVER(rc) ) goto abort_due_to_error; + assert( u.bp.pC->deferredMoveto==0 ); + assert( u.bp.pC->isTable==0 ); + if( !u.bp.pC->nullRow ){ + rc = sqlite3VdbeIdxRowid(db, u.bp.pCrsr, &u.bp.rowid); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; } - } -#endif /* !defined(SQLITE_OMIT_TRIGGER) */ - - /* - ** Perhaps the name is a reference to the ROWID - */ - if( cnt==0 && cntTab==1 && sqlite3IsRowid(zCol) ){ - cnt = 1; - pExpr->iColumn = -1; /* IMP: R-44911-55124 */ - pExpr->affinity = SQLITE_AFF_INTEGER; - } - - /* - ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z - ** might refer to an result-set alias. This happens, for example, when - ** we are resolving names in the WHERE clause of the following command: - ** - ** SELECT a+b AS x FROM table WHERE x<10; - ** - ** In cases like this, replace pExpr with a copy of the expression that - ** forms the result set entry ("a+b" in the example) and return immediately. - ** Note that the expression in the result set should have already been - ** resolved by the time the WHERE clause is resolved. - */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ - for(j=0; jnExpr; j++){ - char *zAs = pEList->a[j].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ - Expr *pOrig; - assert( pExpr->pLeft==0 && pExpr->pRight==0 ); - assert( pExpr->x.pList==0 ); - assert( pExpr->x.pSelect==0 ); - pOrig = pEList->a[j].pExpr; - if( !pNC->allowAgg && ExprHasProperty(pOrig, EP_Agg) ){ - sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); - return WRC_Abort; - } - resolveAlias(pParse, pEList, j, pExpr, ""); - cnt = 1; - pMatch = 0; - assert( zTab==0 && zDb==0 ); - goto lookupname_end; - } - } - } - - /* Advance to the next name context. The loop will exit when either - ** we have a match (cnt>0) or when we run out of name contexts. - */ - if( cnt==0 ){ - pNC = pNC->pNext; + pOut->u.i = u.bp.rowid; + pOut->flags = MEM_Int; } } + break; +} - /* - ** If X and Y are NULL (in other words if only the column name Z is - ** supplied) and the value of Z is enclosed in double-quotes, then - ** Z is a string literal if it doesn't match any column names. In that - ** case, we need to return right away and not make any changes to - ** pExpr. - ** - ** Because no reference was made to outer contexts, the pNC->nRef - ** fields are not changed in any context. - */ - if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ - pExpr->op = TK_STRING; - pExpr->pTab = 0; - return WRC_Prune; - } +/* Opcode: IdxGE P1 P2 P3 P4 P5 +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the ROWID. Compare this key value against the index +** that P1 is currently pointing to, ignoring the ROWID on the P1 index. +** +** If the P1 index entry is greater than or equal to the key value +** then jump to P2. Otherwise fall through to the next instruction. +** +** If P5 is non-zero then the key value is increased by an epsilon +** prior to the comparison. This make the opcode work like IdxGT except +** that if the key from register P3 is a prefix of the key in the cursor, +** the result is false whereas it would be true with IdxGT. +*/ +/* Opcode: IdxLT P1 P2 P3 P4 P5 +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the ROWID. Compare this key value against the index +** that P1 is currently pointing to, ignoring the ROWID on the P1 index. +** +** If the P1 index entry is less than the key value then jump to P2. +** Otherwise fall through to the next instruction. +** +** If P5 is non-zero then the key value is increased by an epsilon prior +** to the comparison. This makes the opcode work like IdxLE. +*/ +case OP_IdxLT: /* jump */ +case OP_IdxGE: { /* jump */ +#if 0 /* local variables moved into u.bq */ + VdbeCursor *pC; + int res; + UnpackedRecord r; +#endif /* local variables moved into u.bq */ - /* - ** cnt==0 means there was not match. cnt>1 means there were two or - ** more matches. Either way, we have an error. - */ - if( cnt!=1 ){ - const char *zErr; - zErr = cnt==0 ? "no such column" : "ambiguous column name"; - if( zDb ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); - }else if( zTab ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + assert( pOp->p1>=0 && pOp->p1nCursor ); + u.bq.pC = p->apCsr[pOp->p1]; + assert( u.bq.pC!=0 ); + assert( u.bq.pC->isOrdered ); + if( ALWAYS(u.bq.pC->pCursor!=0) ){ + assert( u.bq.pC->deferredMoveto==0 ); + assert( pOp->p5==0 || pOp->p5==1 ); + assert( pOp->p4type==P4_INT32 ); + u.bq.r.pKeyInfo = u.bq.pC->pKeyInfo; + u.bq.r.nField = (u16)pOp->p4.i; + if( pOp->p5 ){ + u.bq.r.flags = UNPACKED_INCRKEY | UNPACKED_IGNORE_ROWID; }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); + u.bq.r.flags = UNPACKED_IGNORE_ROWID; + } + u.bq.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iopcode==OP_IdxLT ){ + u.bq.res = -u.bq.res; + }else{ + assert( pOp->opcode==OP_IdxGE ); + u.bq.res++; + } + if( u.bq.res>0 ){ + pc = pOp->p2 - 1 ; } - pParse->checkSchema = 1; - pTopNC->nErr++; } + break; +} - /* If a column from a table in pSrcList is referenced, then record - ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes - ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the - ** column number is greater than the number of bits in the bitmask - ** then set the high-order bit of the bitmask. - */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ - int n = pExpr->iColumn; - testcase( n==BMS-1 ); - if( n>=BMS ){ - n = BMS-1; +/* Opcode: Destroy P1 P2 P3 * * +** +** Delete an entire database table or index whose root page in the database +** file is given by P1. +** +** The table being destroyed is in the main database file if P3==0. If +** P3==1 then the table to be clear is in the auxiliary database file +** that is used to store tables create using CREATE TEMPORARY TABLE. +** +** If AUTOVACUUM is enabled then it is possible that another root page +** might be moved into the newly deleted root page in order to keep all +** root pages contiguous at the beginning of the database. The former +** value of the root page that moved - its value before the move occurred - +** is stored in register P2. If no page +** movement was required (because the table being dropped was already +** the last one in the database) then a zero is stored in register P2. +** If AUTOVACUUM is disabled then a zero is stored in register P2. +** +** See also: Clear +*/ +case OP_Destroy: { /* out2-prerelease */ +#if 0 /* local variables moved into u.br */ + int iMoved; + int iCnt; + Vdbe *pVdbe; + int iDb; +#endif /* local variables moved into u.br */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + u.br.iCnt = 0; + for(u.br.pVdbe=db->pVdbe; u.br.pVdbe; u.br.pVdbe = u.br.pVdbe->pNext){ + if( u.br.pVdbe->magic==VDBE_MAGIC_RUN && u.br.pVdbe->inVtabMethod<2 && u.br.pVdbe->pc>=0 ){ + u.br.iCnt++; } - assert( pMatch->iCursor==pExpr->iTable ); - pMatch->colUsed |= ((Bitmask)1)<pLeft); - pExpr->pLeft = 0; - sqlite3ExprDelete(db, pExpr->pRight); - pExpr->pRight = 0; - pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN); -lookupname_end: - if( cnt==1 ){ - assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); - /* Increment the nRef value on all name contexts from TopNC up to - ** the point where the name matched. */ - for(;;){ - assert( pTopNC!=0 ); - pTopNC->nRef++; - if( pTopNC==pNC ) break; - pTopNC = pTopNC->pNext; +#else + u.br.iCnt = db->activeVdbeCnt; +#endif + pOut->flags = MEM_Null; + if( u.br.iCnt>1 ){ + rc = SQLITE_LOCKED; + p->errorAction = OE_Abort; + }else{ + u.br.iDb = pOp->p3; + assert( u.br.iCnt==1 ); + assert( (p->btreeMask & (((yDbMask)1)<aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved); + pOut->flags = MEM_Int; + pOut->u.i = u.br.iMoved; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( rc==SQLITE_OK && u.br.iMoved!=0 ){ + sqlite3RootPageMoved(db, u.br.iDb, u.br.iMoved, pOp->p1); + /* All OP_Destroy operations occur on the same btree */ + assert( resetSchemaOnFault==0 || resetSchemaOnFault==u.br.iDb+1 ); + resetSchemaOnFault = u.br.iDb+1; } - return WRC_Prune; - } else { - return WRC_Abort; +#endif } + break; } -/* -** Allocate and return a pointer to an expression to load the column iCol -** from datasource iSrc in SrcList pSrc. +/* Opcode: Clear P1 P2 P3 +** +** Delete all contents of the database table or index whose root page +** in the database file is given by P1. But, unlike Destroy, do not +** remove the table or index from the database file. +** +** The table being clear is in the main database file if P2==0. If +** P2==1 then the table to be clear is in the auxiliary database file +** that is used to store tables create using CREATE TEMPORARY TABLE. +** +** If the P3 value is non-zero, then the table referred to must be an +** intkey table (an SQL table, not an index). In this case the row change +** count is incremented by the number of rows in the table being cleared. +** If P3 is greater than zero, then the value stored in register P3 is +** also incremented by the number of rows in the table being cleared. +** +** See also: Destroy */ -SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ - Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); - if( p ){ - struct SrcList_item *pItem = &pSrc->a[iSrc]; - p->pTab = pItem->pTab; - p->iTable = pItem->iCursor; - if( p->pTab->iPKey==iCol ){ - p->iColumn = -1; - }else{ - p->iColumn = (ynVar)iCol; - testcase( iCol==BMS ); - testcase( iCol==BMS-1 ); - pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); +case OP_Clear: { +#if 0 /* local variables moved into u.bs */ + int nChange; +#endif /* local variables moved into u.bs */ + + u.bs.nChange = 0; + assert( (p->btreeMask & (((yDbMask)1)<p2))!=0 ); + rc = sqlite3BtreeClearTable( + db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0) + ); + if( pOp->p3 ){ + p->nChange += u.bs.nChange; + if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); + aMem[pOp->p3].u.i += u.bs.nChange; } - ExprSetProperty(p, EP_Resolved); } - return p; + break; } -/* -** This routine is callback for sqlite3WalkExpr(). +/* Opcode: CreateTable P1 P2 * * * ** -** Resolve symbolic names into TK_COLUMN operators for the current -** node in the expression tree. Return 0 to continue the search down -** the tree or 2 to abort the tree walk. +** Allocate a new table in the main database file if P1==0 or in the +** auxiliary database file if P1==1 or in an attached database if +** P1>1. Write the root page number of the new table into +** register P2 ** -** This routine also does error checking and name resolution for -** function names. The operator for aggregate functions is changed -** to TK_AGG_FUNCTION. +** The difference between a table and an index is this: A table must +** have a 4-byte integer key and can have arbitrary data. An index +** has an arbitrary key but no data. +** +** See also: CreateIndex */ -static int resolveExprStep(Walker *pWalker, Expr *pExpr){ - NameContext *pNC; - Parse *pParse; +/* Opcode: CreateIndex P1 P2 * * * +** +** Allocate a new index in the main database file if P1==0 or in the +** auxiliary database file if P1==1 or in an attached database if +** P1>1. Write the root page number of the new table into +** register P2. +** +** See documentation on OP_CreateTable for additional information. +*/ +case OP_CreateIndex: /* out2-prerelease */ +case OP_CreateTable: { /* out2-prerelease */ +#if 0 /* local variables moved into u.bt */ + int pgno; + int flags; + Db *pDb; +#endif /* local variables moved into u.bt */ - pNC = pWalker->u.pNC; - assert( pNC!=0 ); - pParse = pNC->pParse; - assert( pParse==pWalker->pParse ); + u.bt.pgno = 0; + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + u.bt.pDb = &db->aDb[pOp->p1]; + assert( u.bt.pDb->pBt!=0 ); + if( pOp->opcode==OP_CreateTable ){ + /* u.bt.flags = BTREE_INTKEY; */ + u.bt.flags = BTREE_INTKEY; + }else{ + u.bt.flags = BTREE_BLOBKEY; + } + rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags); + pOut->u.i = u.bt.pgno; + break; +} - if( ExprHasAnyProperty(pExpr, EP_Resolved) ) return WRC_Prune; - ExprSetProperty(pExpr, EP_Resolved); -#ifndef NDEBUG - if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ - SrcList *pSrcList = pNC->pSrcList; - int i; - for(i=0; ipSrcList->nSrc; i++){ - assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); - } +/* Opcode: ParseSchema P1 * * P4 * +** +** Read and parse all entries from the SQLITE_MASTER table of database P1 +** that match the WHERE clause P4. +** +** This opcode invokes the parser to create a new virtual machine, +** then runs the new virtual machine. It is thus a re-entrant opcode. +*/ +case OP_ParseSchema: { +#if 0 /* local variables moved into u.bu */ + int iDb; + const char *zMaster; + char *zSql; + InitData initData; +#endif /* local variables moved into u.bu */ + + /* Any prepared statement that invokes this opcode will hold mutexes + ** on every btree. This is a prerequisite for invoking + ** sqlite3InitCallback(). + */ +#ifdef SQLITE_DEBUG + for(u.bu.iDb=0; u.bu.iDbnDb; u.bu.iDb++){ + assert( u.bu.iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[u.bu.iDb].pBt) ); } #endif - switch( pExpr->op ){ -#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) - /* The special operator TK_ROW means use the rowid for the first - ** column in the FROM clause. This is used by the LIMIT and ORDER BY - ** clause processing on UPDATE and DELETE statements. - */ - case TK_ROW: { - SrcList *pSrcList = pNC->pSrcList; - struct SrcList_item *pItem; - assert( pSrcList && pSrcList->nSrc==1 ); - pItem = pSrcList->a; - pExpr->op = TK_COLUMN; - pExpr->pTab = pItem->pTab; - pExpr->iTable = pItem->iCursor; - pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; - break; - } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ - - /* A lone identifier is the name of a column. - */ - case TK_ID: { - return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); - } - - /* A table name and column name: ID.ID - ** Or a database, table and column: ID.ID.ID - */ - case TK_DOT: { - const char *zColumn; - const char *zTable; - const char *zDb; - Expr *pRight; - - /* if( pSrcList==0 ) break; */ - pRight = pExpr->pRight; - if( pRight->op==TK_ID ){ - zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; - }else{ - assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; - } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); - } - - /* Resolve function names - */ - case TK_CONST_FUNC: - case TK_FUNCTION: { - ExprList *pList = pExpr->x.pList; /* The argument list */ - int n = pList ? pList->nExpr : 0; /* Number of arguments */ - int no_such_func = 0; /* True if no such function exists */ - int wrong_num_args = 0; /* True if wrong number of arguments */ - int is_agg = 0; /* True if is an aggregate function */ - int auth; /* Authorization to use the function */ - int nId; /* Number of characters in function name */ - const char *zId; /* The function name. */ - FuncDef *pDef; /* Information about the function */ - u8 enc = ENC(pParse->db); /* The database encoding */ - - testcase( pExpr->op==TK_CONST_FUNC ); - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); - if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -1, enc, 0); - if( pDef==0 ){ - no_such_func = 1; - }else{ - wrong_num_args = 1; - } - }else{ - is_agg = pDef->xFunc==0; - } -#ifndef SQLITE_OMIT_AUTHORIZATION - if( pDef ){ - auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; - } - pExpr->op = TK_NULL; - return WRC_Prune; - } - } -#endif - if( is_agg && !pNC->allowAgg ){ - sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); - pNC->nErr++; - is_agg = 0; - }else if( no_such_func ){ - sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); - pNC->nErr++; - }else if( wrong_num_args ){ - sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", - nId, zId); - pNC->nErr++; - } - if( is_agg ){ - pExpr->op = TK_AGG_FUNCTION; - pNC->hasAgg = 1; - } - if( is_agg ) pNC->allowAgg = 0; - sqlite3WalkExprList(pWalker, pList); - if( is_agg ) pNC->allowAgg = 1; - /* FIX ME: Compute pExpr->affinity based on the expected return - ** type of the function - */ - return WRC_Prune; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_SELECT: - case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); -#endif - case TK_IN: { - testcase( pExpr->op==TK_IN ); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - int nRef = pNC->nRef; -#ifndef SQLITE_OMIT_CHECK - if( pNC->isCheck ){ - sqlite3ErrorMsg(pParse,"subqueries prohibited in CHECK constraints"); - } -#endif - sqlite3WalkSelect(pWalker, pExpr->x.pSelect); - assert( pNC->nRef>=nRef ); - if( nRef!=pNC->nRef ){ - ExprSetProperty(pExpr, EP_VarSelect); - } - } - break; - } -#ifndef SQLITE_OMIT_CHECK - case TK_VARIABLE: { - if( pNC->isCheck ){ - sqlite3ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); - } - break; + u.bu.iDb = pOp->p1; + assert( u.bu.iDb>=0 && u.bu.iDbnDb ); + assert( DbHasProperty(db, u.bu.iDb, DB_SchemaLoaded) ); + /* Used to be a conditional */ { + u.bu.zMaster = SCHEMA_TABLE(u.bu.iDb); + u.bu.initData.db = db; + u.bu.initData.iDb = pOp->p1; + u.bu.initData.pzErrMsg = &p->zErrMsg; + u.bu.zSql = sqlite3MPrintf(db, + "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", + db->aDb[u.bu.iDb].zName, u.bu.zMaster, pOp->p4.z); + if( u.bu.zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + assert( db->init.busy==0 ); + db->init.busy = 1; + u.bu.initData.rc = SQLITE_OK; + assert( !db->mallocFailed ); + rc = sqlite3_exec(db, u.bu.zSql, sqlite3InitCallback, &u.bu.initData, 0); + if( rc==SQLITE_OK ) rc = u.bu.initData.rc; + sqlite3DbFree(db, u.bu.zSql); + db->init.busy = 0; } -#endif } - return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; + if( rc==SQLITE_NOMEM ){ + goto no_mem; + } + break; } -/* -** pEList is a list of expressions which are really the result set of the -** a SELECT statement. pE is a term in an ORDER BY or GROUP BY clause. -** This routine checks to see if pE is a simple identifier which corresponds -** to the AS-name of one of the terms of the expression list. If it is, -** this routine return an integer between 1 and N where N is the number of -** elements in pEList, corresponding to the matching entry. If there is -** no match, or if pE is not a simple identifier, then this routine -** return 0. +#if !defined(SQLITE_OMIT_ANALYZE) +/* Opcode: LoadAnalysis P1 * * * * ** -** pEList has been resolved. pE has not. +** Read the sqlite_stat1 table for database P1 and load the content +** of that table into the internal index hash table. This will cause +** the analysis to be used when preparing all subsequent queries. */ -static int resolveAsName( - Parse *pParse, /* Parsing context for error messages */ - ExprList *pEList, /* List of expressions to scan */ - Expr *pE /* Expression we are trying to match */ -){ - int i; /* Loop counter */ - - UNUSED_PARAMETER(pParse); - - if( pE->op==TK_ID ){ - char *zCol = pE->u.zToken; - for(i=0; inExpr; i++){ - char *zAs = pEList->a[i].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ - return i+1; - } - } - } - return 0; +case OP_LoadAnalysis: { + assert( pOp->p1>=0 && pOp->p1nDb ); + rc = sqlite3AnalysisLoad(db, pOp->p1); + break; } +#endif /* !defined(SQLITE_OMIT_ANALYZE) */ -/* -** pE is a pointer to an expression which is a single term in the -** ORDER BY of a compound SELECT. The expression has not been -** name resolved. -** -** At the point this routine is called, we already know that the -** ORDER BY term is not an integer index into the result set. That -** case is handled by the calling routine. -** -** Attempt to match pE against result set columns in the left-most -** SELECT statement. Return the index i of the matching column, -** as an indication to the caller that it should sort by the i-th column. -** The left-most column is 1. In other words, the value returned is the -** same integer value that would be used in the SQL statement to indicate -** the column. +/* Opcode: DropTable P1 * * P4 * ** -** If there is no match, return 0. Return -1 if an error occurs. +** Remove the internal (in-memory) data structures that describe +** the table named P4 in database P1. This is called after a table +** is dropped in order to keep the internal representation of the +** schema consistent with what is on disk. */ -static int resolveOrderByTermToExprList( - Parse *pParse, /* Parsing context for error messages */ - Select *pSelect, /* The SELECT statement with the ORDER BY clause */ - Expr *pE /* The specific ORDER BY term */ -){ - int i; /* Loop counter */ - ExprList *pEList; /* The columns of the result set */ - NameContext nc; /* Name context for resolving pE */ - sqlite3 *db; /* Database connection */ - int rc; /* Return code from subprocedures */ - u8 savedSuppErr; /* Saved value of db->suppressErr */ - - assert( sqlite3ExprIsInteger(pE, &i)==0 ); - pEList = pSelect->pEList; - - /* Resolve all names in the ORDER BY term expression - */ - memset(&nc, 0, sizeof(nc)); - nc.pParse = pParse; - nc.pSrcList = pSelect->pSrc; - nc.pEList = pEList; - nc.allowAgg = 1; - nc.nErr = 0; - db = pParse->db; - savedSuppErr = db->suppressErr; - db->suppressErr = 1; - rc = sqlite3ResolveExprNames(&nc, pE); - db->suppressErr = savedSuppErr; - if( rc ) return 0; - - /* Try to match the ORDER BY expression against an expression - ** in the result set. Return an 1-based index of the matching - ** result-set entry. - */ - for(i=0; inExpr; i++){ - if( sqlite3ExprCompare(pEList->a[i].pExpr, pE)<2 ){ - return i+1; - } - } +case OP_DropTable: { + sqlite3UnlinkAndDeleteTable(db, pOp->p1, pOp->p4.z); + break; +} - /* If no match, return 0. */ - return 0; +/* Opcode: DropIndex P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the index named P4 in database P1. This is called after an index +** is dropped in order to keep the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropIndex: { + sqlite3UnlinkAndDeleteIndex(db, pOp->p1, pOp->p4.z); + break; } -/* -** Generate an ORDER BY or GROUP BY term out-of-range error. +/* Opcode: DropTrigger P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the trigger named P4 in database P1. This is called after a trigger +** is dropped in order to keep the internal representation of the +** schema consistent with what is on disk. */ -static void resolveOutOfRangeError( - Parse *pParse, /* The error context into which to write the error */ - const char *zType, /* "ORDER" or "GROUP" */ - int i, /* The index (1-based) of the term out of range */ - int mx /* Largest permissible value of i */ -){ - sqlite3ErrorMsg(pParse, - "%r %s BY term out of range - should be " - "between 1 and %d", i, zType, mx); +case OP_DropTrigger: { + sqlite3UnlinkAndDeleteTrigger(db, pOp->p1, pOp->p4.z); + break; } -/* -** Analyze the ORDER BY clause in a compound SELECT statement. Modify -** each term of the ORDER BY clause is a constant integer between 1 -** and N where N is the number of columns in the compound SELECT. + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* Opcode: IntegrityCk P1 P2 P3 * P5 ** -** ORDER BY terms that are already an integer between 1 and N are -** unmodified. ORDER BY terms that are integers outside the range of -** 1 through N generate an error. ORDER BY terms that are expressions -** are matched against result set expressions of compound SELECT -** beginning with the left-most SELECT and working toward the right. -** At the first match, the ORDER BY expression is transformed into -** the integer column number. +** Do an analysis of the currently open database. Store in +** register P1 the text of an error message describing any problems. +** If no problems are found, store a NULL in register P1. ** -** Return the number of errors seen. +** The register P3 contains the maximum number of allowed errors. +** At most reg(P3) errors will be reported. +** In other words, the analysis stops as soon as reg(P1) errors are +** seen. Reg(P1) is updated with the number of errors remaining. +** +** The root page numbers of all tables in the database are integer +** stored in reg(P1), reg(P1+1), reg(P1+2), .... There are P2 tables +** total. +** +** If P5 is not zero, the check is done on the auxiliary database +** file, not the main database file. +** +** This opcode is used to implement the integrity_check pragma. */ -static int resolveCompoundOrderBy( - Parse *pParse, /* Parsing context. Leave error messages here */ - Select *pSelect /* The SELECT statement containing the ORDER BY */ -){ - int i; - ExprList *pOrderBy; - ExprList *pEList; - sqlite3 *db; - int moreToDo = 1; +case OP_IntegrityCk: { +#if 0 /* local variables moved into u.bv */ + int nRoot; /* Number of tables to check. (Number of root pages.) */ + int *aRoot; /* Array of rootpage numbers for tables to be checked */ + int j; /* Loop counter */ + int nErr; /* Number of errors reported */ + char *z; /* Text of the error report */ + Mem *pnErr; /* Register keeping track of errors remaining */ +#endif /* local variables moved into u.bv */ - pOrderBy = pSelect->pOrderBy; - if( pOrderBy==0 ) return 0; - db = pParse->db; -#if SQLITE_MAX_COLUMN - if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ - sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); - return 1; - } -#endif - for(i=0; inExpr; i++){ - pOrderBy->a[i].done = 0; - } - pSelect->pNext = 0; - while( pSelect->pPrior ){ - pSelect->pPrior->pNext = pSelect; - pSelect = pSelect->pPrior; - } - while( pSelect && moreToDo ){ - struct ExprList_item *pItem; - moreToDo = 0; - pEList = pSelect->pEList; - assert( pEList!=0 ); - for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ - int iCol = -1; - Expr *pE, *pDup; - if( pItem->done ) continue; - pE = pItem->pExpr; - if( sqlite3ExprIsInteger(pE, &iCol) ){ - if( iCol<=0 || iCol>pEList->nExpr ){ - resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); - return 1; - } - }else{ - iCol = resolveAsName(pParse, pEList, pE); - if( iCol==0 ){ - pDup = sqlite3ExprDup(db, pE, 0); - if( !db->mallocFailed ){ - assert(pDup); - iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); - } - sqlite3ExprDelete(db, pDup); - } - } - if( iCol>0 ){ - CollSeq *pColl = pE->pColl; - int flags = pE->flags & EP_ExpCollate; - sqlite3ExprDelete(db, pE); - pItem->pExpr = pE = sqlite3Expr(db, TK_INTEGER, 0); - if( pE==0 ) return 1; - pE->pColl = pColl; - pE->flags |= EP_IntValue | flags; - pE->u.iValue = iCol; - pItem->iCol = (u16)iCol; - pItem->done = 1; - }else{ - moreToDo = 1; - } - } - pSelect = pSelect->pNext; + u.bv.nRoot = pOp->p2; + assert( u.bv.nRoot>0 ); + u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) ); + if( u.bv.aRoot==0 ) goto no_mem; + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + u.bv.pnErr = &aMem[pOp->p3]; + assert( (u.bv.pnErr->flags & MEM_Int)!=0 ); + assert( (u.bv.pnErr->flags & (MEM_Str|MEM_Blob))==0 ); + pIn1 = &aMem[pOp->p1]; + for(u.bv.j=0; u.bv.jnExpr; i++){ - if( pOrderBy->a[i].done==0 ){ - sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " - "column in the result set", i+1); - return 1; - } + u.bv.aRoot[u.bv.j] = 0; + assert( pOp->p5nDb ); + assert( (p->btreeMask & (((yDbMask)1)<p5))!=0 ); + u.bv.z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, u.bv.aRoot, u.bv.nRoot, + (int)u.bv.pnErr->u.i, &u.bv.nErr); + sqlite3DbFree(db, u.bv.aRoot); + u.bv.pnErr->u.i -= u.bv.nErr; + sqlite3VdbeMemSetNull(pIn1); + if( u.bv.nErr==0 ){ + assert( u.bv.z==0 ); + }else if( u.bv.z==0 ){ + goto no_mem; + }else{ + sqlite3VdbeMemSetStr(pIn1, u.bv.z, -1, SQLITE_UTF8, sqlite3_free); } - return 0; + UPDATE_MAX_BLOBSIZE(pIn1); + sqlite3VdbeChangeEncoding(pIn1, encoding); + break; } +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ -/* -** Check every term in the ORDER BY or GROUP BY clause pOrderBy of -** the SELECT statement pSelect. If any term is reference to a -** result set expression (as determined by the ExprList.a.iCol field) -** then convert that term into a copy of the corresponding result set -** column. +/* Opcode: RowSetAdd P1 P2 * * * ** -** If any errors are detected, add an error message to pParse and -** return non-zero. Return zero if no errors are seen. +** Insert the integer value held by register P2 into a boolean index +** held in register P1. +** +** An assertion fails if P2 is not an integer. */ -SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( - Parse *pParse, /* Parsing context. Leave error messages here */ - Select *pSelect, /* The SELECT statement containing the clause */ - ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ - const char *zType /* "ORDER" or "GROUP" */ -){ - int i; - sqlite3 *db = pParse->db; - ExprList *pEList; - struct ExprList_item *pItem; - - if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; -#if SQLITE_MAX_COLUMN - if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ - sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); - return 1; - } -#endif - pEList = pSelect->pEList; - assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ - for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ - if( pItem->iCol ){ - if( pItem->iCol>pEList->nExpr ){ - resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); - return 1; - } - resolveAlias(pParse, pEList, pItem->iCol-1, pItem->pExpr, zType); - } +case OP_RowSetAdd: { /* in1, in2 */ + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Int)!=0 ); + if( (pIn1->flags & MEM_RowSet)==0 ){ + sqlite3VdbeMemSetRowSet(pIn1); + if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; } - return 0; + sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); + break; } -/* -** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. -** The Name context of the SELECT statement is pNC. zType is either -** "ORDER" or "GROUP" depending on which type of clause pOrderBy is. -** -** This routine resolves each term of the clause into an expression. -** If the order-by term is an integer I between 1 and N (where N is the -** number of columns in the result set of the SELECT) then the expression -** in the resolution is a copy of the I-th result-set expression. If -** the order-by term is an identify that corresponds to the AS-name of -** a result-set expression, then the term resolves to a copy of the -** result-set expression. Otherwise, the expression is resolved in -** the usual way - using sqlite3ResolveExprNames(). +/* Opcode: RowSetRead P1 P2 P3 * * ** -** This routine returns the number of errors. If errors occur, then -** an appropriate error message might be left in pParse. (OOM errors -** excepted.) +** Extract the smallest value from boolean index P1 and put that value into +** register P3. Or, if boolean index P1 is initially empty, leave P3 +** unchanged and jump to instruction P2. */ -static int resolveOrderGroupBy( - NameContext *pNC, /* The name context of the SELECT statement */ - Select *pSelect, /* The SELECT statement holding pOrderBy */ - ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ - const char *zType /* Either "ORDER" or "GROUP", as appropriate */ -){ - int i; /* Loop counter */ - int iCol; /* Column number */ - struct ExprList_item *pItem; /* A term of the ORDER BY clause */ - Parse *pParse; /* Parsing context */ - int nResult; /* Number of terms in the result set */ +case OP_RowSetRead: { /* jump, in1, out3 */ +#if 0 /* local variables moved into u.bw */ + i64 val; +#endif /* local variables moved into u.bw */ + CHECK_FOR_INTERRUPT; + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_RowSet)==0 + || sqlite3RowSetNext(pIn1->u.pRowSet, &u.bw.val)==0 + ){ + /* The boolean index is empty */ + sqlite3VdbeMemSetNull(pIn1); + pc = pOp->p2 - 1; + }else{ + /* A value was pulled from the index */ + sqlite3VdbeMemSetInt64(&aMem[pOp->p3], u.bw.val); + } + break; +} - if( pOrderBy==0 ) return 0; - nResult = pSelect->pEList->nExpr; - pParse = pNC->pParse; - for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ - Expr *pE = pItem->pExpr; - iCol = resolveAsName(pParse, pSelect->pEList, pE); - if( iCol>0 ){ - /* If an AS-name match is found, mark this ORDER BY column as being - ** a copy of the iCol-th result-set column. The subsequent call to - ** sqlite3ResolveOrderGroupBy() will convert the expression to a - ** copy of the iCol-th result-set expression. */ - pItem->iCol = (u16)iCol; - continue; - } - if( sqlite3ExprIsInteger(pE, &iCol) ){ - /* The ORDER BY term is an integer constant. Again, set the column - ** number so that sqlite3ResolveOrderGroupBy() will convert the - ** order-by term to a copy of the result-set expression */ - if( iCol<1 ){ - resolveOutOfRangeError(pParse, zType, i+1, nResult); - return 1; - } - pItem->iCol = (u16)iCol; - continue; - } +/* Opcode: RowSetTest P1 P2 P3 P4 +** +** Register P3 is assumed to hold a 64-bit integer value. If register P1 +** contains a RowSet object and that RowSet object contains +** the value held in P3, jump to register P2. Otherwise, insert the +** integer in P3 into the RowSet and continue on to the +** next opcode. +** +** The RowSet object is optimized for the case where successive sets +** of integers, where each set contains no duplicates. Each set +** of values is identified by a unique P4 value. The first set +** must have P4==0, the final set P4=-1. P4 must be either -1 or +** non-negative. For non-negative values of P4 only the lower 4 +** bits are significant. +** +** This allows optimizations: (a) when P4==0 there is no need to test +** the rowset object for P3, as it is guaranteed not to contain it, +** (b) when P4==-1 there is no need to insert the value, as it will +** never be tested for, and (c) when a value that is part of set X is +** inserted, there is no need to search to see if the same value was +** previously inserted as part of set X (only if it was previously +** inserted as part of some other set). +*/ +case OP_RowSetTest: { /* jump, in1, in3 */ +#if 0 /* local variables moved into u.bx */ + int iSet; + int exists; +#endif /* local variables moved into u.bx */ - /* Otherwise, treat the ORDER BY term as an ordinary expression */ - pItem->iCol = 0; - if( sqlite3ResolveExprNames(pNC, pE) ){ - return 1; + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + u.bx.iSet = pOp->p4.i; + assert( pIn3->flags&MEM_Int ); + + /* If there is anything other than a rowset object in memory cell P1, + ** delete it now and initialize P1 with an empty rowset + */ + if( (pIn1->flags & MEM_RowSet)==0 ){ + sqlite3VdbeMemSetRowSet(pIn1); + if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + } + + assert( pOp->p4type==P4_INT32 ); + assert( u.bx.iSet==-1 || u.bx.iSet>=0 ); + if( u.bx.iSet ){ + u.bx.exists = sqlite3RowSetTest(pIn1->u.pRowSet, + (u8)(u.bx.iSet>=0 ? u.bx.iSet & 0xf : 0xff), + pIn3->u.i); + if( u.bx.exists ){ + pc = pOp->p2 - 1; + break; } } - return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); + if( u.bx.iSet>=0 ){ + sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); + } + break; } -/* -** Resolve names in the SELECT statement p and all of its descendents. + +#ifndef SQLITE_OMIT_TRIGGER + +/* Opcode: Program P1 P2 P3 P4 * +** +** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). +** +** P1 contains the address of the memory cell that contains the first memory +** cell in an array of values used as arguments to the sub-program. P2 +** contains the address to jump to if the sub-program throws an IGNORE +** exception using the RAISE() function. Register P3 contains the address +** of a memory cell in this (the parent) VM that is used to allocate the +** memory required by the sub-vdbe at runtime. +** +** P4 is a pointer to the VM containing the trigger program. */ -static int resolveSelectStep(Walker *pWalker, Select *p){ - NameContext *pOuterNC; /* Context that contains this SELECT */ - NameContext sNC; /* Name context of this SELECT */ - int isCompound; /* True if p is a compound select */ - int nCompound; /* Number of compound terms processed so far */ - Parse *pParse; /* Parsing context */ - ExprList *pEList; /* Result set expression list */ - int i; /* Loop counter */ - ExprList *pGroupBy; /* The GROUP BY clause */ - Select *pLeftmost; /* Left-most of SELECT of a compound */ - sqlite3 *db; /* Database connection */ - +case OP_Program: { /* jump */ +#if 0 /* local variables moved into u.by */ + int nMem; /* Number of memory registers for sub-program */ + int nByte; /* Bytes of runtime space required for sub-program */ + Mem *pRt; /* Register to allocate runtime space */ + Mem *pMem; /* Used to iterate through memory cells */ + Mem *pEnd; /* Last memory cell in new array */ + VdbeFrame *pFrame; /* New vdbe frame to execute in */ + SubProgram *pProgram; /* Sub-program to execute */ + void *t; /* Token identifying trigger */ +#endif /* local variables moved into u.by */ - assert( p!=0 ); - if( p->selFlags & SF_Resolved ){ - return WRC_Prune; - } - pOuterNC = pWalker->u.pNC; - pParse = pWalker->pParse; - db = pParse->db; + u.by.pProgram = pOp->p4.pProgram; + u.by.pRt = &aMem[pOp->p3]; + assert( memIsValid(u.by.pRt) ); + assert( u.by.pProgram->nOp>0 ); - /* Normally sqlite3SelectExpand() will be called first and will have - ** already expanded this SELECT. However, if this is a subquery within - ** an expression, sqlite3ResolveExprNames() will be called without a - ** prior call to sqlite3SelectExpand(). When that happens, let - ** sqlite3SelectPrep() do all of the processing for this SELECT. - ** sqlite3SelectPrep() will invoke both sqlite3SelectExpand() and - ** this routine in the correct order. - */ - if( (p->selFlags & SF_Expanded)==0 ){ - sqlite3SelectPrep(pParse, p, pOuterNC); - return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + /* If the p5 flag is clear, then recursive invocation of triggers is + ** disabled for backwards compatibility (p5 is set if this sub-program + ** is really a trigger, not a foreign key action, and the flag set + ** and cleared by the "PRAGMA recursive_triggers" command is clear). + ** + ** It is recursive invocation of triggers, at the SQL level, that is + ** disabled. In some cases a single trigger may generate more than one + ** SubProgram (if the trigger may be executed with more than one different + ** ON CONFLICT algorithm). SubProgram structures associated with a + ** single trigger all have the same value for the SubProgram.token + ** variable. */ + if( pOp->p5 ){ + u.by.t = u.by.pProgram->token; + for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent); + if( u.by.pFrame ) break; } - isCompound = p->pPrior!=0; - nCompound = 0; - pLeftmost = p; - while( p ){ - assert( (p->selFlags & SF_Expanded)!=0 ); - assert( (p->selFlags & SF_Resolved)==0 ); - p->selFlags |= SF_Resolved; + if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ + rc = SQLITE_ERROR; + sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); + break; + } - /* Resolve the expressions in the LIMIT and OFFSET clauses. These - ** are not allowed to refer to any names, so pass an empty NameContext. - */ - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - if( sqlite3ResolveExprNames(&sNC, p->pLimit) || - sqlite3ResolveExprNames(&sNC, p->pOffset) ){ - return WRC_Abort; - } - - /* Set up the local name-context to pass to sqlite3ResolveExprNames() to - ** resolve the result-set expression list. - */ - sNC.allowAgg = 1; - sNC.pSrcList = p->pSrc; - sNC.pNext = pOuterNC; - - /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } - - /* Recursively resolve names in all subqueries - */ - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - if( pItem->pSelect ){ - const char *zSavedContext = pParse->zAuthContext; - if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); - pParse->zAuthContext = zSavedContext; - if( pParse->nErr || db->mallocFailed ) return WRC_Abort; - } - } - - /* If there are no aggregate functions in the result-set, and no GROUP BY - ** expression, do not allow aggregates in any of the other expressions. - */ - assert( (p->selFlags & SF_Aggregate)==0 ); - pGroupBy = p->pGroupBy; - if( pGroupBy || sNC.hasAgg ){ - p->selFlags |= SF_Aggregate; - }else{ - sNC.allowAgg = 0; - } - - /* If a HAVING clause is present, then there must be a GROUP BY clause. - */ - if( p->pHaving && !pGroupBy ){ - sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); - return WRC_Abort; - } - - /* Add the expression list to the name-context before parsing the - ** other expressions in the SELECT statement. This is so that - ** expressions in the WHERE clause (etc.) can refer to expressions by - ** aliases in the result set. - ** - ** Minor point: If this is the case, then the expression will be - ** re-evaluated for each reference to it. + /* Register u.by.pRt is used to store the memory required to save the state + ** of the current program, and the memory required at runtime to execute + ** the trigger program. If this trigger has been fired before, then u.by.pRt + ** is already allocated. Otherwise, it must be initialized. */ + if( (u.by.pRt->flags&MEM_Frame)==0 ){ + /* SubProgram.nMem is set to the number of memory cells used by the + ** program stored in SubProgram.aOp. As well as these, one memory + ** cell is required for each cursor used by the program. Set local + ** variable u.by.nMem (and later, VdbeFrame.nChildMem) to this value. */ - sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; + u.by.nMem = u.by.pProgram->nMem + u.by.pProgram->nCsr; + u.by.nByte = ROUND8(sizeof(VdbeFrame)) + + u.by.nMem * sizeof(Mem) + + u.by.pProgram->nCsr * sizeof(VdbeCursor *); + u.by.pFrame = sqlite3DbMallocZero(db, u.by.nByte); + if( !u.by.pFrame ){ + goto no_mem; } + sqlite3VdbeMemRelease(u.by.pRt); + u.by.pRt->flags = MEM_Frame; + u.by.pRt->u.pFrame = u.by.pFrame; - /* The ORDER BY and GROUP BY clauses may not refer to terms in - ** outer queries - */ - sNC.pNext = 0; - sNC.allowAgg = 1; + u.by.pFrame->v = p; + u.by.pFrame->nChildMem = u.by.nMem; + u.by.pFrame->nChildCsr = u.by.pProgram->nCsr; + u.by.pFrame->pc = pc; + u.by.pFrame->aMem = p->aMem; + u.by.pFrame->nMem = p->nMem; + u.by.pFrame->apCsr = p->apCsr; + u.by.pFrame->nCursor = p->nCursor; + u.by.pFrame->aOp = p->aOp; + u.by.pFrame->nOp = p->nOp; + u.by.pFrame->token = u.by.pProgram->token; - /* Process the ORDER BY clause for singleton SELECT statements. - ** The ORDER BY clause for compounds SELECT statements is handled - ** below, after all of the result-sets for all of the elements of - ** the compound have been resolved. - */ - if( !isCompound && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ - return WRC_Abort; - } - if( db->mallocFailed ){ - return WRC_Abort; - } - - /* Resolve the GROUP BY clause. At the same time, make sure - ** the GROUP BY clause does not contain aggregate functions. - */ - if( pGroupBy ){ - struct ExprList_item *pItem; - - if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ - return WRC_Abort; - } - for(i=0, pItem=pGroupBy->a; inExpr; i++, pItem++){ - if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ - sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " - "the GROUP BY clause"); - return WRC_Abort; - } - } + u.by.pEnd = &VdbeFrameMem(u.by.pFrame)[u.by.pFrame->nChildMem]; + for(u.by.pMem=VdbeFrameMem(u.by.pFrame); u.by.pMem!=u.by.pEnd; u.by.pMem++){ + u.by.pMem->flags = MEM_Null; + u.by.pMem->db = db; } - - /* Advance to the next term of the compound - */ - p = p->pPrior; - nCompound++; + }else{ + u.by.pFrame = u.by.pRt->u.pFrame; + assert( u.by.pProgram->nMem+u.by.pProgram->nCsr==u.by.pFrame->nChildMem ); + assert( u.by.pProgram->nCsr==u.by.pFrame->nChildCsr ); + assert( pc==u.by.pFrame->pc ); } - /* Resolve the ORDER BY on a compound SELECT after all terms of - ** the compound have been resolved. - */ - if( isCompound && resolveCompoundOrderBy(pParse, pLeftmost) ){ - return WRC_Abort; - } + p->nFrame++; + u.by.pFrame->pParent = p->pFrame; + u.by.pFrame->lastRowid = lastRowid; + u.by.pFrame->nChange = p->nChange; + p->nChange = 0; + p->pFrame = u.by.pFrame; + p->aMem = aMem = &VdbeFrameMem(u.by.pFrame)[-1]; + p->nMem = u.by.pFrame->nChildMem; + p->nCursor = (u16)u.by.pFrame->nChildCsr; + p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; + p->aOp = aOp = u.by.pProgram->aOp; + p->nOp = u.by.pProgram->nOp; + pc = -1; - return WRC_Prune; + break; } -/* -** This routine walks an expression tree and resolves references to -** table columns and result-set columns. At the same time, do error -** checking on function usage and set a flag if any aggregate functions -** are seen. +/* Opcode: Param P1 P2 * * * ** -** To resolve table columns references we look for nodes (or subtrees) of the -** form X.Y.Z or Y.Z or just Z where +** This opcode is only ever present in sub-programs called via the +** OP_Program instruction. Copy a value currently stored in a memory +** cell of the calling (parent) frame to cell P2 in the current frames +** address space. This is used by trigger programs to access the new.* +** and old.* values. ** -** X: The name of a database. Ex: "main" or "temp" or -** the symbolic name assigned to an ATTACH-ed database. +** The address of the cell in the parent frame is determined by adding +** the value of the P1 argument to the value of the P1 argument to the +** calling OP_Program instruction. +*/ +case OP_Param: { /* out2-prerelease */ +#if 0 /* local variables moved into u.bz */ + VdbeFrame *pFrame; + Mem *pIn; +#endif /* local variables moved into u.bz */ + u.bz.pFrame = p->pFrame; + u.bz.pIn = &u.bz.pFrame->aMem[pOp->p1 + u.bz.pFrame->aOp[u.bz.pFrame->pc].p1]; + sqlite3VdbeMemShallowCopy(pOut, u.bz.pIn, MEM_Ephem); + break; +} + +#endif /* #ifndef SQLITE_OMIT_TRIGGER */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +/* Opcode: FkCounter P1 P2 * * * ** -** Y: The name of a table in a FROM clause. Or in a trigger -** one of the special names "old" or "new". +** Increment a "constraint counter" by P2 (P2 may be negative or positive). +** If P1 is non-zero, the database constraint counter is incremented +** (deferred foreign key constraints). Otherwise, if P1 is zero, the +** statement counter is incremented (immediate foreign key constraints). +*/ +case OP_FkCounter: { + if( pOp->p1 ){ + db->nDeferredCons += pOp->p2; + }else{ + p->nFkConstraint += pOp->p2; + } + break; +} + +/* Opcode: FkIfZero P1 P2 * * * ** -** Z: The name of a column in table Y. +** This opcode tests if a foreign key constraint-counter is currently zero. +** If so, jump to instruction P2. Otherwise, fall through to the next +** instruction. ** -** The node at the root of the subtree is modified as follows: +** If P1 is non-zero, then the jump is taken if the database constraint-counter +** is zero (the one that counts deferred constraint violations). If P1 is +** zero, the jump is taken if the statement constraint-counter is zero +** (immediate foreign key constraint violations). +*/ +case OP_FkIfZero: { /* jump */ + if( pOp->p1 ){ + if( db->nDeferredCons==0 ) pc = pOp->p2-1; + }else{ + if( p->nFkConstraint==0 ) pc = pOp->p2-1; + } + break; +} +#endif /* #ifndef SQLITE_OMIT_FOREIGN_KEY */ + +#ifndef SQLITE_OMIT_AUTOINCREMENT +/* Opcode: MemMax P1 P2 * * * ** -** Expr.op Changed to TK_COLUMN -** Expr.pTab Points to the Table object for X.Y -** Expr.iColumn The column index in X.Y. -1 for the rowid. -** Expr.iTable The VDBE cursor number for X.Y +** P1 is a register in the root frame of this VM (the root frame is +** different from the current frame if this instruction is being executed +** within a sub-program). Set the value of register P1 to the maximum of +** its current value and the value in register P2. ** +** This instruction throws an error if the memory cell is not initially +** an integer. +*/ +case OP_MemMax: { /* in2 */ +#if 0 /* local variables moved into u.ca */ + Mem *pIn1; + VdbeFrame *pFrame; +#endif /* local variables moved into u.ca */ + if( p->pFrame ){ + for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent); + u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1]; + }else{ + u.ca.pIn1 = &aMem[pOp->p1]; + } + assert( memIsValid(u.ca.pIn1) ); + sqlite3VdbeMemIntegerify(u.ca.pIn1); + pIn2 = &aMem[pOp->p2]; + sqlite3VdbeMemIntegerify(pIn2); + if( u.ca.pIn1->u.iu.i){ + u.ca.pIn1->u.i = pIn2->u.i; + } + break; +} +#endif /* SQLITE_OMIT_AUTOINCREMENT */ + +/* Opcode: IfPos P1 P2 * * * ** -** To resolve result-set references, look for expression nodes of the -** form Z (with no X and Y prefix) where the Z matches the right-hand -** size of an AS clause in the result-set of a SELECT. The Z expression -** is replaced by a copy of the left-hand side of the result-set expression. -** Table-name and function resolution occurs on the substituted expression -** tree. For example, in: +** If the value of register P1 is 1 or greater, jump to P2. ** -** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY x; +** It is illegal to use this instruction on a register that does +** not contain an integer. An assertion fault will result if you try. +*/ +case OP_IfPos: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + if( pIn1->u.i>0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: IfNeg P1 P2 * * * ** -** The "x" term of the order by is replaced by "a+b" to render: +** If the value of register P1 is less than zero, jump to P2. ** -** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; +** It is illegal to use this instruction on a register that does +** not contain an integer. An assertion fault will result if you try. +*/ +case OP_IfNeg: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + if( pIn1->u.i<0 ){ + pc = pOp->p2 - 1; + } + break; +} + +/* Opcode: IfZero P1 P2 P3 * * ** -** Function calls are checked to make sure that the function is -** defined and that the correct number of arguments are specified. -** If the function is an aggregate function, then the pNC->hasAgg is -** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. -** If an expression contains aggregate functions then the EP_Agg -** property on the expression is set. +** The register P1 must contain an integer. Add literal P3 to the +** value in register P1. If the result is exactly 0, jump to P2. ** -** An error message is left in pParse if anything is amiss. The number -** if errors is returned. +** It is illegal to use this instruction on a register that does +** not contain an integer. An assertion fault will result if you try. */ -SQLITE_PRIVATE int sqlite3ResolveExprNames( - NameContext *pNC, /* Namespace to resolve expressions in. */ - Expr *pExpr /* The expression to be analyzed. */ -){ - int savedHasAgg; - Walker w; - - if( pExpr==0 ) return 0; -#if SQLITE_MAX_EXPR_DEPTH>0 - { - Parse *pParse = pNC->pParse; - if( sqlite3ExprCheckHeight(pParse, pExpr->nHeight+pNC->pParse->nHeight) ){ - return 1; - } - pParse->nHeight += pExpr->nHeight; - } -#endif - savedHasAgg = pNC->hasAgg; - pNC->hasAgg = 0; - w.xExprCallback = resolveExprStep; - w.xSelectCallback = resolveSelectStep; - w.pParse = pNC->pParse; - w.u.pNC = pNC; - sqlite3WalkExpr(&w, pExpr); -#if SQLITE_MAX_EXPR_DEPTH>0 - pNC->pParse->nHeight -= pExpr->nHeight; -#endif - if( pNC->nErr>0 || w.pParse->nErr>0 ){ - ExprSetProperty(pExpr, EP_Error); - } - if( pNC->hasAgg ){ - ExprSetProperty(pExpr, EP_Agg); - }else if( savedHasAgg ){ - pNC->hasAgg = 1; +case OP_IfZero: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; + if( pIn1->u.i==0 ){ + pc = pOp->p2 - 1; } - return ExprHasProperty(pExpr, EP_Error); + break; } - -/* -** Resolve all names in all expressions of a SELECT and in all -** decendents of the SELECT, including compounds off of p->pPrior, -** subqueries in expressions, and subqueries used as FROM clause -** terms. +/* Opcode: AggStep * P2 P3 P4 P5 ** -** See sqlite3ResolveExprNames() for a description of the kinds of -** transformations that occur. +** Execute the step function for an aggregate. The +** function has P5 arguments. P4 is a pointer to the FuncDef +** structure that specifies the function. Use register +** P3 as the accumulator. ** -** All SELECT statements should have been expanded using -** sqlite3SelectExpand() prior to invoking this routine. +** The P5 arguments are taken from register P2 and its +** successors. */ -SQLITE_PRIVATE void sqlite3ResolveSelectNames( - Parse *pParse, /* The parser context */ - Select *p, /* The SELECT statement being coded. */ - NameContext *pOuterNC /* Name context for parent SELECT statement */ -){ - Walker w; +case OP_AggStep: { +#if 0 /* local variables moved into u.cb */ + int n; + int i; + Mem *pMem; + Mem *pRec; + sqlite3_context ctx; + sqlite3_value **apVal; +#endif /* local variables moved into u.cb */ - assert( p!=0 ); - w.xExprCallback = resolveExprStep; - w.xSelectCallback = resolveSelectStep; - w.pParse = pParse; - w.u.pNC = pOuterNC; - sqlite3WalkSelect(&w, p); + u.cb.n = pOp->p5; + assert( u.cb.n>=0 ); + u.cb.pRec = &aMem[pOp->p2]; + u.cb.apVal = p->apArg; + assert( u.cb.apVal || u.cb.n==0 ); + for(u.cb.i=0; u.cb.ip4.pFunc; + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3]; + u.cb.pMem->n++; + u.cb.ctx.s.flags = MEM_Null; + u.cb.ctx.s.z = 0; + u.cb.ctx.s.zMalloc = 0; + u.cb.ctx.s.xDel = 0; + u.cb.ctx.s.db = db; + u.cb.ctx.isError = 0; + u.cb.ctx.pColl = 0; + if( u.cb.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ + assert( pOp>p->aOp ); + assert( pOp[-1].p4type==P4_COLLSEQ ); + assert( pOp[-1].opcode==OP_CollSeq ); + u.cb.ctx.pColl = pOp[-1].p4.pColl; + } + (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); /* IMP: R-24505-23230 */ + if( u.cb.ctx.isError ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s)); + rc = u.cb.ctx.isError; + } + + sqlite3VdbeMemRelease(&u.cb.ctx.s); + + break; } -/************** End of resolve.c *********************************************/ -/************** Begin file expr.c ********************************************/ -/* -** 2001 September 15 +/* Opcode: AggFinal P1 P2 * P4 * ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** Execute the finalizer function for an aggregate. P1 is +** the memory location that is the accumulator for the aggregate. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the degenerate case where +** the step function was not previously called. +*/ +case OP_AggFinal: { +#if 0 /* local variables moved into u.cc */ + Mem *pMem; +#endif /* local variables moved into u.cc */ + assert( pOp->p1>0 && pOp->p1<=p->nMem ); + u.cc.pMem = &aMem[pOp->p1]; + assert( (u.cc.pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); + rc = sqlite3VdbeMemFinalize(u.cc.pMem, pOp->p4.pFunc); + if( rc ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cc.pMem)); + } + sqlite3VdbeChangeEncoding(u.cc.pMem, encoding); + UPDATE_MAX_BLOBSIZE(u.cc.pMem); + if( sqlite3VdbeMemTooBig(u.cc.pMem) ){ + goto too_big; + } + break; +} + +#ifndef SQLITE_OMIT_WAL +/* Opcode: Checkpoint P1 P2 P3 * * ** -************************************************************************* -** This file contains routines used for analyzing expressions and -** for generating VDBE code that evaluates expressions in SQLite. +** Checkpoint database P1. This is a no-op if P1 is not currently in +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL +** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** SQLITE_BUSY or not, respectively. Write the number of pages in the +** WAL after the checkpoint into mem[P3+1] and the number of pages +** in the WAL that have been checkpointed after the checkpoint +** completes into mem[P3+2]. However on an error, mem[P3+1] and +** mem[P3+2] are initialized to -1. */ +case OP_Checkpoint: { +#if 0 /* local variables moved into u.cd */ + int i; /* Loop counter */ + int aRes[3]; /* Results */ + Mem *pMem; /* Write results here */ +#endif /* local variables moved into u.cd */ -/* -** Return the 'affinity' of the expression pExpr if any. + u.cd.aRes[0] = 0; + u.cd.aRes[1] = u.cd.aRes[2] = -1; + assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE + || pOp->p2==SQLITE_CHECKPOINT_FULL + || pOp->p2==SQLITE_CHECKPOINT_RESTART + ); + rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &u.cd.aRes[1], &u.cd.aRes[2]); + if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + u.cd.aRes[0] = 1; + } + for(u.cd.i=0, u.cd.pMem = &aMem[pOp->p3]; u.cd.i<3; u.cd.i++, u.cd.pMem++){ + sqlite3VdbeMemSetInt64(u.cd.pMem, (i64)u.cd.aRes[u.cd.i]); + } + break; +}; +#endif + +#ifndef SQLITE_OMIT_PRAGMA +/* Opcode: JournalMode P1 P2 P3 * P5 ** -** If pExpr is a column, a reference to a column via an 'AS' alias, -** or a sub-select with a column as the return value, then the -** affinity of that column is returned. Otherwise, 0x00 is returned, -** indicating no affinity for the expression. +** Change the journal mode of database P1 to P3. P3 must be one of the +** PAGER_JOURNALMODE_XXX values. If changing between the various rollback +** modes (delete, truncate, persist, off and memory), this is a simple +** operation. No IO is required. ** -** i.e. the WHERE clause expresssions in the following statements all -** have an affinity: +** If changing into or out of WAL mode the procedure is more complicated. ** -** CREATE TABLE t1(a); -** SELECT * FROM t1 WHERE a; -** SELECT a AS b FROM t1 WHERE b; -** SELECT * FROM t1 WHERE (select a from t1); +** Write a string containing the final journal-mode to register P2. */ -SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ - int op = pExpr->op; - if( op==TK_SELECT ){ - assert( pExpr->flags&EP_xIsSelect ); - return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); - } -#ifndef SQLITE_OMIT_CAST - if( op==TK_CAST ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - return sqlite3AffinityType(pExpr->u.zToken); +case OP_JournalMode: { /* out2-prerelease */ +#if 0 /* local variables moved into u.ce */ + Btree *pBt; /* Btree to change journal mode of */ + Pager *pPager; /* Pager associated with pBt */ + int eNew; /* New journal mode */ + int eOld; /* The old journal mode */ + const char *zFilename; /* Name of database file for pPager */ +#endif /* local variables moved into u.ce */ + + u.ce.eNew = pOp->p3; + assert( u.ce.eNew==PAGER_JOURNALMODE_DELETE + || u.ce.eNew==PAGER_JOURNALMODE_TRUNCATE + || u.ce.eNew==PAGER_JOURNALMODE_PERSIST + || u.ce.eNew==PAGER_JOURNALMODE_OFF + || u.ce.eNew==PAGER_JOURNALMODE_MEMORY + || u.ce.eNew==PAGER_JOURNALMODE_WAL + || u.ce.eNew==PAGER_JOURNALMODE_QUERY + ); + assert( pOp->p1>=0 && pOp->p1nDb ); + + u.ce.pBt = db->aDb[pOp->p1].pBt; + u.ce.pPager = sqlite3BtreePager(u.ce.pBt); + u.ce.eOld = sqlite3PagerGetJournalMode(u.ce.pPager); + if( u.ce.eNew==PAGER_JOURNALMODE_QUERY ) u.ce.eNew = u.ce.eOld; + if( !sqlite3PagerOkToChangeJournalMode(u.ce.pPager) ) u.ce.eNew = u.ce.eOld; + +#ifndef SQLITE_OMIT_WAL + u.ce.zFilename = sqlite3PagerFilename(u.ce.pPager); + + /* Do not allow a transition to journal_mode=WAL for a database + ** in temporary storage or if the VFS does not support shared memory + */ + if( u.ce.eNew==PAGER_JOURNALMODE_WAL + && (u.ce.zFilename[0]==0 /* Temp file */ + || !sqlite3PagerWalSupported(u.ce.pPager)) /* No shared-memory support */ + ){ + u.ce.eNew = u.ce.eOld; } -#endif - if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) - && pExpr->pTab!=0 + + if( (u.ce.eNew!=u.ce.eOld) + && (u.ce.eOld==PAGER_JOURNALMODE_WAL || u.ce.eNew==PAGER_JOURNALMODE_WAL) ){ - /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = pExpr->iColumn; - if( j<0 ) return SQLITE_AFF_INTEGER; - assert( pExpr->pTab && jpTab->nCol ); - return pExpr->pTab->aCol[j].affinity; + if( !db->autoCommit || db->activeVdbeCnt>1 ){ + rc = SQLITE_ERROR; + sqlite3SetString(&p->zErrMsg, db, + "cannot change %s wal mode from within a transaction", + (u.ce.eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") + ); + break; + }else{ + + if( u.ce.eOld==PAGER_JOURNALMODE_WAL ){ + /* If leaving WAL mode, close the log file. If successful, the call + ** to PagerCloseWal() checkpoints and deletes the write-ahead-log + ** file. An EXCLUSIVE lock may still be held on the database file + ** after a successful return. + */ + rc = sqlite3PagerCloseWal(u.ce.pPager); + if( rc==SQLITE_OK ){ + sqlite3PagerSetJournalMode(u.ce.pPager, u.ce.eNew); + } + }else if( u.ce.eOld==PAGER_JOURNALMODE_MEMORY ){ + /* Cannot transition directly from MEMORY to WAL. Use mode OFF + ** as an intermediate */ + sqlite3PagerSetJournalMode(u.ce.pPager, PAGER_JOURNALMODE_OFF); + } + + /* Open a transaction on the database file. Regardless of the journal + ** mode, this transaction always uses a rollback journal. + */ + assert( sqlite3BtreeIsInTrans(u.ce.pBt)==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeSetVersion(u.ce.pBt, (u.ce.eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); + } + } } - return pExpr->affinity; -} +#endif /* ifndef SQLITE_OMIT_WAL */ -/* -** Set the explicit collating sequence for an expression to the -** collating sequence supplied in the second argument. -*/ -SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ - if( pExpr && pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; + if( rc ){ + u.ce.eNew = u.ce.eOld; } - return pExpr; -} + u.ce.eNew = sqlite3PagerSetJournalMode(u.ce.pPager, u.ce.eNew); -/* -** Set the collating sequence for expression pExpr to be the collating -** sequence named by pToken. Return a pointer to the revised expression. -** The collating sequence is marked as "explicit" using the EP_ExpCollate -** flag. An explicit collating sequence will override implicit -** collating sequences. + pOut = &aMem[pOp->p2]; + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = (char *)sqlite3JournalModename(u.ce.eNew); + pOut->n = sqlite3Strlen30(pOut->z); + pOut->enc = SQLITE_UTF8; + sqlite3VdbeChangeEncoding(pOut, encoding); + break; +}; +#endif /* SQLITE_OMIT_PRAGMA */ + +#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) +/* Opcode: Vacuum * * * * * +** +** Vacuum the entire database. This opcode will cause other virtual +** machines to be created and run. It may not be called from within +** a transaction. */ -SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ - char *zColl = 0; /* Dequoted name of collation sequence */ - CollSeq *pColl; - sqlite3 *db = pParse->db; - zColl = sqlite3NameFromToken(db, pCollName); - pColl = sqlite3LocateCollSeq(pParse, zColl); - sqlite3ExprSetColl(pExpr, pColl); - sqlite3DbFree(db, zColl); - return pExpr; +case OP_Vacuum: { + rc = sqlite3RunVacuum(&p->zErrMsg, db); + break; } +#endif -/* -** Return the default collation sequence for the expression pExpr. If -** there is no default collation type, return 0. +#if !defined(SQLITE_OMIT_AUTOVACUUM) +/* Opcode: IncrVacuum P1 P2 * * * +** +** Perform a single step of the incremental vacuum procedure on +** the P1 database. If the vacuum has finished, jump to instruction +** P2. Otherwise, fall through to the next instruction. */ -SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ - CollSeq *pColl = 0; - Expr *p = pExpr; - while( ALWAYS(p) ){ - int op; - pColl = p->pColl; - if( pColl ) break; - op = p->op; - if( p->pTab!=0 && ( - op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER - )){ - /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - const char *zColl; - int j = p->iColumn; - if( j>=0 ){ - sqlite3 *db = pParse->db; - zColl = p->pTab->aCol[j].zColl; - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - pExpr->pColl = pColl; - } - break; - } - if( op!=TK_CAST && op!=TK_UPLUS ){ - break; - } - p = p->pLeft; - } - if( sqlite3CheckCollSeq(pParse, pColl) ){ - pColl = 0; +case OP_IncrVacuum: { /* jump */ +#if 0 /* local variables moved into u.cf */ + Btree *pBt; +#endif /* local variables moved into u.cf */ + + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + u.cf.pBt = db->aDb[pOp->p1].pBt; + rc = sqlite3BtreeIncrVacuum(u.cf.pBt); + if( rc==SQLITE_DONE ){ + pc = pOp->p2 - 1; + rc = SQLITE_OK; } - return pColl; + break; } +#endif -/* -** pExpr is an operand of a comparison operator. aff2 is the -** type affinity of the other operand. This routine returns the -** type affinity that should be used for the comparison operator. +/* Opcode: Expire P1 * * * * +** +** Cause precompiled statements to become expired. An expired statement +** fails with an error code of SQLITE_SCHEMA if it is ever executed +** (via sqlite3_step()). +** +** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, +** then only the currently executing statement is affected. */ -SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ - char aff1 = sqlite3ExprAffinity(pExpr); - if( aff1 && aff2 ){ - /* Both sides of the comparison are columns. If one has numeric - ** affinity, use that. Otherwise use no affinity. - */ - if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ - return SQLITE_AFF_NUMERIC; - }else{ - return SQLITE_AFF_NONE; - } - }else if( !aff1 && !aff2 ){ - /* Neither side of the comparison is a column. Compare the - ** results directly. - */ - return SQLITE_AFF_NONE; +case OP_Expire: { + if( !pOp->p1 ){ + sqlite3ExpirePreparedStatements(db); }else{ - /* One side is a column, the other is not. Use the columns affinity. */ - assert( aff1==0 || aff2==0 ); - return (aff1 + aff2); + p->expired = 1; } + break; } -/* -** pExpr is a comparison operator. Return the type affinity that should -** be applied to both operands prior to doing the comparison. +#ifndef SQLITE_OMIT_SHARED_CACHE +/* Opcode: TableLock P1 P2 P3 P4 * +** +** Obtain a lock on a particular table. This instruction is only used when +** the shared-cache feature is enabled. +** +** P1 is the index of the database in sqlite3.aDb[] of the database +** on which the lock is acquired. A readlock is obtained if P3==0 or +** a write lock if P3==1. +** +** P2 contains the root-page of the table to lock. +** +** P4 contains a pointer to the name of the table being locked. This is only +** used to generate an error message if the lock cannot be obtained. */ -static char comparisonAffinity(Expr *pExpr){ - char aff; - assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || - pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || - pExpr->op==TK_NE || pExpr->op==TK_IS || pExpr->op==TK_ISNOT ); - assert( pExpr->pLeft ); - aff = sqlite3ExprAffinity(pExpr->pLeft); - if( pExpr->pRight ){ - aff = sqlite3CompareAffinity(pExpr->pRight, aff); - }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( !aff ){ - aff = SQLITE_AFF_NONE; +case OP_TableLock: { + u8 isWriteLock = (u8)pOp->p3; + if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ + int p1 = pOp->p1; + assert( p1>=0 && p1nDb ); + assert( (p->btreeMask & (((yDbMask)1)<aDb[p1].pBt, pOp->p2, isWriteLock); + if( (rc&0xFF)==SQLITE_LOCKED ){ + const char *z = pOp->p4.z; + sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); + } } - return aff; + break; } +#endif /* SQLITE_OMIT_SHARED_CACHE */ -/* -** pExpr is a comparison expression, eg. '=', '<', IN(...) etc. -** idx_affinity is the affinity of an indexed column. Return true -** if the index with affinity idx_affinity may be used to implement -** the comparison in pExpr. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VBegin * * * P4 * +** +** P4 may be a pointer to an sqlite3_vtab structure. If so, call the +** xBegin method for that table. +** +** Also, whether or not P4 is set, check that this is not being called from +** within a callback to a virtual table xSync() method. If it is, the error +** code will be set to SQLITE_LOCKED. */ -SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ - char aff = comparisonAffinity(pExpr); - switch( aff ){ - case SQLITE_AFF_NONE: - return 1; - case SQLITE_AFF_TEXT: - return idx_affinity==SQLITE_AFF_TEXT; - default: - return sqlite3IsNumericAffinity(idx_affinity); - } +case OP_VBegin: { +#if 0 /* local variables moved into u.cg */ + VTable *pVTab; +#endif /* local variables moved into u.cg */ + u.cg.pVTab = pOp->p4.pVtab; + rc = sqlite3VtabBegin(db, u.cg.pVTab); + if( u.cg.pVTab ) importVtabErrMsg(p, u.cg.pVTab->pVtab); + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Return the P5 value that should be used for a binary comparison -** opcode (OP_Eq, OP_Ge etc.) used to compare pExpr1 and pExpr2. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VCreate P1 * * P4 * +** +** P4 is the name of a virtual table in database P1. Call the xCreate method +** for that table. */ -static u8 binaryCompareP5(Expr *pExpr1, Expr *pExpr2, int jumpIfNull){ - u8 aff = (char)sqlite3ExprAffinity(pExpr2); - aff = (u8)sqlite3CompareAffinity(pExpr1, aff) | (u8)jumpIfNull; - return aff; +case OP_VCreate: { + rc = sqlite3VtabCallCreate(db, pOp->p1, pOp->p4.z, &p->zErrMsg); + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Return a pointer to the collation sequence that should be used by -** a binary comparison operator comparing pLeft and pRight. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VDestroy P1 * * P4 * ** -** If the left hand expression has a collating sequence type, then it is -** used. Otherwise the collation sequence for the right hand expression -** is used, or the default (BINARY) if neither expression has a collating -** type. +** P4 is the name of a virtual table in database P1. Call the xDestroy method +** of that table. +*/ +case OP_VDestroy: { + p->inVtabMethod = 2; + rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); + p->inVtabMethod = 0; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VOpen P1 * * P4 * ** -** Argument pRight (but not pLeft) may be a null pointer. In this case, -** it is not considered. +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** P1 is a cursor number. This opcode opens a cursor to the virtual +** table and stores that cursor in P1. */ -SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq( - Parse *pParse, - Expr *pLeft, - Expr *pRight -){ - CollSeq *pColl; - assert( pLeft ); - if( pLeft->flags & EP_ExpCollate ){ - assert( pLeft->pColl ); - pColl = pLeft->pColl; - }else if( pRight && pRight->flags & EP_ExpCollate ){ - assert( pRight->pColl ); - pColl = pRight->pColl; - }else{ - pColl = sqlite3ExprCollSeq(pParse, pLeft); - if( !pColl ){ - pColl = sqlite3ExprCollSeq(pParse, pRight); +case OP_VOpen: { +#if 0 /* local variables moved into u.ch */ + VdbeCursor *pCur; + sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab *pVtab; + sqlite3_module *pModule; +#endif /* local variables moved into u.ch */ + + u.ch.pCur = 0; + u.ch.pVtabCursor = 0; + u.ch.pVtab = pOp->p4.pVtab->pVtab; + u.ch.pModule = (sqlite3_module *)u.ch.pVtab->pModule; + assert(u.ch.pVtab && u.ch.pModule); + rc = u.ch.pModule->xOpen(u.ch.pVtab, &u.ch.pVtabCursor); + importVtabErrMsg(p, u.ch.pVtab); + if( SQLITE_OK==rc ){ + /* Initialize sqlite3_vtab_cursor base class */ + u.ch.pVtabCursor->pVtab = u.ch.pVtab; + + /* Initialise vdbe cursor object */ + u.ch.pCur = allocateCursor(p, pOp->p1, 0, -1, 0); + if( u.ch.pCur ){ + u.ch.pCur->pVtabCursor = u.ch.pVtabCursor; + u.ch.pCur->pModule = u.ch.pVtabCursor->pVtab->pModule; + }else{ + db->mallocFailed = 1; + u.ch.pModule->xClose(u.ch.pVtabCursor); } } - return pColl; + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Generate code for a comparison operator. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VFilter P1 P2 P3 P4 * +** +** P1 is a cursor opened using VOpen. P2 is an address to jump to if +** the filtered result set is empty. +** +** P4 is either NULL or a string that was generated by the xBestIndex +** method of the module. The interpretation of the P4 string is left +** to the module implementation. +** +** This opcode invokes the xFilter method on the virtual table specified +** by P1. The integer query plan parameter to xFilter is stored in register +** P3. Register P3+1 stores the argc parameter to be passed to the +** xFilter method. Registers P3+2..P3+1+argc are the argc +** additional parameters which are passed to +** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter. +** +** A jump is made to P2 if the result set after filtering would be empty. */ -static int codeCompare( - Parse *pParse, /* The parsing (and code generating) context */ - Expr *pLeft, /* The left operand */ - Expr *pRight, /* The right operand */ - int opcode, /* The comparison opcode */ - int in1, int in2, /* Register holding operands */ - int dest, /* Jump here if true. */ - int jumpIfNull /* If true, jump if either operand is NULL */ -){ - int p5; - int addr; - CollSeq *p4; +case OP_VFilter: { /* jump */ +#if 0 /* local variables moved into u.ci */ + int nArg; + int iQuery; + const sqlite3_module *pModule; + Mem *pQuery; + Mem *pArgc; + sqlite3_vtab_cursor *pVtabCursor; + sqlite3_vtab *pVtab; + VdbeCursor *pCur; + int res; + int i; + Mem **apArg; +#endif /* local variables moved into u.ci */ - p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); - p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); - addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, - (void*)p4, P4_COLLSEQ); - sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); - return addr; -} + u.ci.pQuery = &aMem[pOp->p3]; + u.ci.pArgc = &u.ci.pQuery[1]; + u.ci.pCur = p->apCsr[pOp->p1]; + assert( memIsValid(u.ci.pQuery) ); + REGISTER_TRACE(pOp->p3, u.ci.pQuery); + assert( u.ci.pCur->pVtabCursor ); + u.ci.pVtabCursor = u.ci.pCur->pVtabCursor; + u.ci.pVtab = u.ci.pVtabCursor->pVtab; + u.ci.pModule = u.ci.pVtab->pModule; -#if SQLITE_MAX_EXPR_DEPTH>0 -/* -** Check that argument nHeight is less than or equal to the maximum -** expression depth allowed. If it is not, leave an error message in -** pParse. -*/ -SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ - int rc = SQLITE_OK; - int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; - if( nHeight>mxHeight ){ - sqlite3ErrorMsg(pParse, - "Expression tree is too large (maximum depth %d)", mxHeight - ); - rc = SQLITE_ERROR; + /* Grab the index number and argc parameters */ + assert( (u.ci.pQuery->flags&MEM_Int)!=0 && u.ci.pArgc->flags==MEM_Int ); + u.ci.nArg = (int)u.ci.pArgc->u.i; + u.ci.iQuery = (int)u.ci.pQuery->u.i; + + /* Invoke the xFilter method */ + { + u.ci.res = 0; + u.ci.apArg = p->apArg; + for(u.ci.i = 0; u.ci.iinVtabMethod = 1; + rc = u.ci.pModule->xFilter(u.ci.pVtabCursor, u.ci.iQuery, pOp->p4.z, u.ci.nArg, u.ci.apArg); + p->inVtabMethod = 0; + importVtabErrMsg(p, u.ci.pVtab); + if( rc==SQLITE_OK ){ + u.ci.res = u.ci.pModule->xEof(u.ci.pVtabCursor); + } + + if( u.ci.res ){ + pc = pOp->p2 - 1; + } } - return rc; + u.ci.pCur->nullRow = 0; + + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* The following three functions, heightOfExpr(), heightOfExprList() -** and heightOfSelect(), are used to determine the maximum height -** of any expression tree referenced by the structure passed as the -** first argument. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VColumn P1 P2 P3 * * ** -** If this maximum height is greater than the current value pointed -** to by pnHeight, the second parameter, then set *pnHeight to that -** value. +** Store the value of the P2-th column of +** the row of the virtual-table that the +** P1 cursor is pointing to into register P3. */ -static void heightOfExpr(Expr *p, int *pnHeight){ - if( p ){ - if( p->nHeight>*pnHeight ){ - *pnHeight = p->nHeight; - } +case OP_VColumn: { +#if 0 /* local variables moved into u.cj */ + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + Mem *pDest; + sqlite3_context sContext; +#endif /* local variables moved into u.cj */ + + VdbeCursor *pCur = p->apCsr[pOp->p1]; + assert( pCur->pVtabCursor ); + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + u.cj.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.cj.pDest); + if( pCur->nullRow ){ + sqlite3VdbeMemSetNull(u.cj.pDest); + break; } -} -static void heightOfExprList(ExprList *p, int *pnHeight){ - if( p ){ - int i; - for(i=0; inExpr; i++){ - heightOfExpr(p->a[i].pExpr, pnHeight); - } + u.cj.pVtab = pCur->pVtabCursor->pVtab; + u.cj.pModule = u.cj.pVtab->pModule; + assert( u.cj.pModule->xColumn ); + memset(&u.cj.sContext, 0, sizeof(u.cj.sContext)); + + /* The output cell may already have a buffer allocated. Move + ** the current contents to u.cj.sContext.s so in case the user-function + ** can use the already allocated buffer instead of allocating a + ** new one. + */ + sqlite3VdbeMemMove(&u.cj.sContext.s, u.cj.pDest); + MemSetTypeFlag(&u.cj.sContext.s, MEM_Null); + + rc = u.cj.pModule->xColumn(pCur->pVtabCursor, &u.cj.sContext, pOp->p2); + importVtabErrMsg(p, u.cj.pVtab); + if( u.cj.sContext.isError ){ + rc = u.cj.sContext.isError; } -} -static void heightOfSelect(Select *p, int *pnHeight){ - if( p ){ - heightOfExpr(p->pWhere, pnHeight); - heightOfExpr(p->pHaving, pnHeight); - heightOfExpr(p->pLimit, pnHeight); - heightOfExpr(p->pOffset, pnHeight); - heightOfExprList(p->pEList, pnHeight); - heightOfExprList(p->pGroupBy, pnHeight); - heightOfExprList(p->pOrderBy, pnHeight); - heightOfSelect(p->pPrior, pnHeight); + + /* Copy the result of the function to the P3 register. We + ** do this regardless of whether or not an error occurred to ensure any + ** dynamic allocation in u.cj.sContext.s (a Mem struct) is released. + */ + sqlite3VdbeChangeEncoding(&u.cj.sContext.s, encoding); + sqlite3VdbeMemMove(u.cj.pDest, &u.cj.sContext.s); + REGISTER_TRACE(pOp->p3, u.cj.pDest); + UPDATE_MAX_BLOBSIZE(u.cj.pDest); + + if( sqlite3VdbeMemTooBig(u.cj.pDest) ){ + goto too_big; } + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Set the Expr.nHeight variable in the structure passed as an -** argument. An expression with no children, Expr.pList or -** Expr.pSelect member has a height of 1. Any other expression -** has a height equal to the maximum height of any other -** referenced Expr plus one. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VNext P1 P2 * * * +** +** Advance virtual table P1 to the next row in its result set and +** jump to instruction P2. Or, if the virtual table has reached +** the end of its result set, then fall through to the next instruction. */ -static void exprSetHeight(Expr *p){ - int nHeight = 0; - heightOfExpr(p->pLeft, &nHeight); - heightOfExpr(p->pRight, &nHeight); - if( ExprHasProperty(p, EP_xIsSelect) ){ - heightOfSelect(p->x.pSelect, &nHeight); - }else{ - heightOfExprList(p->x.pList, &nHeight); +case OP_VNext: { /* jump */ +#if 0 /* local variables moved into u.ck */ + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + int res; + VdbeCursor *pCur; +#endif /* local variables moved into u.ck */ + + u.ck.res = 0; + u.ck.pCur = p->apCsr[pOp->p1]; + assert( u.ck.pCur->pVtabCursor ); + if( u.ck.pCur->nullRow ){ + break; } - p->nHeight = nHeight + 1; -} + u.ck.pVtab = u.ck.pCur->pVtabCursor->pVtab; + u.ck.pModule = u.ck.pVtab->pModule; + assert( u.ck.pModule->xNext ); -/* -** Set the Expr.nHeight variable using the exprSetHeight() function. If -** the height is greater than the maximum allowed expression depth, -** leave an error in pParse. -*/ -SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ - exprSetHeight(p); - sqlite3ExprCheckHeight(pParse, p->nHeight); + /* Invoke the xNext() method of the module. There is no way for the + ** underlying implementation to return an error if one occurs during + ** xNext(). Instead, if an error occurs, true is returned (indicating that + ** data is available) and the error code returned when xColumn or + ** some other method is next invoked on the save virtual table cursor. + */ + p->inVtabMethod = 1; + rc = u.ck.pModule->xNext(u.ck.pCur->pVtabCursor); + p->inVtabMethod = 0; + importVtabErrMsg(p, u.ck.pVtab); + if( rc==SQLITE_OK ){ + u.ck.res = u.ck.pModule->xEof(u.ck.pCur->pVtabCursor); + } + + if( !u.ck.res ){ + /* If there is data, jump to P2 */ + pc = pOp->p2 - 1; + } + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Return the maximum height of any expression tree referenced -** by the select statement passed as an argument. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VRename P1 * * P4 * +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** This opcode invokes the corresponding xRename method. The value +** in register P1 is passed as the zName argument to the xRename method. */ -SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ - int nHeight = 0; - heightOfSelect(p, &nHeight); - return nHeight; +case OP_VRename: { +#if 0 /* local variables moved into u.cl */ + sqlite3_vtab *pVtab; + Mem *pName; +#endif /* local variables moved into u.cl */ + + u.cl.pVtab = pOp->p4.pVtab->pVtab; + u.cl.pName = &aMem[pOp->p1]; + assert( u.cl.pVtab->pModule->xRename ); + assert( memIsValid(u.cl.pName) ); + REGISTER_TRACE(pOp->p1, u.cl.pName); + assert( u.cl.pName->flags & MEM_Str ); + rc = u.cl.pVtab->pModule->xRename(u.cl.pVtab, u.cl.pName->z); + importVtabErrMsg(p, u.cl.pVtab); + p->expired = 0; + + break; } -#else - #define exprSetHeight(y) -#endif /* SQLITE_MAX_EXPR_DEPTH>0 */ +#endif -/* -** This routine is the core allocator for Expr nodes. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VUpdate P1 P2 P3 P4 * ** -** Construct a new expression node and return a pointer to it. Memory -** for this node and for the pToken argument is a single allocation -** obtained from sqlite3DbMalloc(). The calling function -** is responsible for making sure the node eventually gets freed. +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** This opcode invokes the corresponding xUpdate method. P2 values +** are contiguous memory cells starting at P3 to pass to the xUpdate +** invocation. The value in register (P3+P2-1) corresponds to the +** p2th element of the argv array passed to xUpdate. ** -** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performance. The deQuote -** parameter is ignored if pToken is NULL or if the token does not -** appear to be quoted. If the quotes were of the form "..." (double-quotes) -** then the EP_DblQuoted flag is set on the expression node. +** The xUpdate method will do a DELETE or an INSERT or both. +** The argv[0] element (which corresponds to memory cell P3) +** is the rowid of a row to delete. If argv[0] is NULL then no +** deletion occurs. The argv[1] element is the rowid of the new +** row. This can be NULL to have the virtual table select the new +** rowid for itself. The subsequent elements in the array are +** the values of columns in the new row. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage -** is allocated to hold the integer text and the dequote flag is ignored. +** If P2==1 then no insert is performed. argv[0] is the rowid of +** a row to delete. +** +** P1 is a boolean flag. If it is set to true and the xUpdate call +** is successful, then the value returned by sqlite3_last_insert_rowid() +** is set to the value of the rowid for the row just inserted. */ -SQLITE_PRIVATE Expr *sqlite3ExprAlloc( - sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ - int op, /* Expression opcode */ - const Token *pToken, /* Token argument. Might be NULL */ - int dequote /* True to dequote */ -){ - Expr *pNew; - int nExtra = 0; - int iValue = 0; +case OP_VUpdate: { +#if 0 /* local variables moved into u.cm */ + sqlite3_vtab *pVtab; + sqlite3_module *pModule; + int nArg; + int i; + sqlite_int64 rowid; + Mem **apArg; + Mem *pX; +#endif /* local variables moved into u.cm */ - if( pToken ){ - if( op!=TK_INTEGER || pToken->z==0 - || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace + ); + u.cm.pVtab = pOp->p4.pVtab->pVtab; + u.cm.pModule = (sqlite3_module *)u.cm.pVtab->pModule; + u.cm.nArg = pOp->p2; + assert( pOp->p4type==P4_VTAB ); + if( ALWAYS(u.cm.pModule->xUpdate) ){ + u8 vtabOnConflict = db->vtabOnConflict; + u.cm.apArg = p->apArg; + u.cm.pX = &aMem[pOp->p3]; + for(u.cm.i=0; u.cm.ivtabOnConflict = pOp->p5; + rc = u.cm.pModule->xUpdate(u.cm.pVtab, u.cm.nArg, u.cm.apArg, &u.cm.rowid); + db->vtabOnConflict = vtabOnConflict; + importVtabErrMsg(p, u.cm.pVtab); + if( rc==SQLITE_OK && pOp->p1 ){ + assert( u.cm.nArg>1 && u.cm.apArg[0] && (u.cm.apArg[0]->flags&MEM_Null) ); + db->lastRowid = lastRowid = u.cm.rowid; } - } - pNew = sqlite3DbMallocZero(db, sizeof(Expr)+nExtra); - if( pNew ){ - pNew->op = (u8)op; - pNew->iAgg = -1; - if( pToken ){ - if( nExtra==0 ){ - pNew->flags |= EP_IntValue; - pNew->u.iValue = iValue; + if( rc==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( pOp->p5==OE_Ignore ){ + rc = SQLITE_OK; }else{ - int c; - pNew->u.zToken = (char*)&pNew[1]; - memcpy(pNew->u.zToken, pToken->z, pToken->n); - pNew->u.zToken[pToken->n] = 0; - if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ - sqlite3Dequote(pNew->u.zToken); - if( c=='"' ) pNew->flags |= EP_DblQuoted; - } + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); } + }else{ + p->nChange++; } -#if SQLITE_MAX_EXPR_DEPTH>0 - pNew->nHeight = 1; -#endif } - return pNew; + break; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Allocate a new expression node from a zero-terminated token that has -** already been dequoted. +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: Pagecount P1 P2 * * * +** +** Write the current number of pages in database P1 to memory cell P2. */ -SQLITE_PRIVATE Expr *sqlite3Expr( - sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ - int op, /* Expression opcode */ - const char *zToken /* Token argument. Might be NULL */ -){ - Token x; - x.z = zToken; - x.n = zToken ? sqlite3Strlen30(zToken) : 0; - return sqlite3ExprAlloc(db, op, &x, 0); +case OP_Pagecount: { /* out2-prerelease */ + pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); + break; } +#endif -/* -** Attach subtrees pLeft and pRight to the Expr node pRoot. + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: MaxPgcnt P1 P2 P3 * * ** -** If pRoot==NULL that means that a memory allocation error has occurred. -** In that case, delete the subtrees pLeft and pRight. +** Try to set the maximum page count for database P1 to the value in P3. +** Do not let the maximum page count fall below the current page count and +** do not change the maximum page count value if P3==0. +** +** Store the maximum page count after the change in register P2. */ -SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( - sqlite3 *db, - Expr *pRoot, - Expr *pLeft, - Expr *pRight -){ - if( pRoot==0 ){ - assert( db->mallocFailed ); - sqlite3ExprDelete(db, pLeft); - sqlite3ExprDelete(db, pRight); - }else{ - if( pRight ){ - pRoot->pRight = pRight; - if( pRight->flags & EP_ExpCollate ){ - pRoot->flags |= EP_ExpCollate; - pRoot->pColl = pRight->pColl; - } - } - if( pLeft ){ - pRoot->pLeft = pLeft; - if( pLeft->flags & EP_ExpCollate ){ - pRoot->flags |= EP_ExpCollate; - pRoot->pColl = pLeft->pColl; - } - } - exprSetHeight(pRoot); +case OP_MaxPgcnt: { /* out2-prerelease */ + unsigned int newMax; + Btree *pBt; + + pBt = db->aDb[pOp->p1].pBt; + newMax = 0; + if( pOp->p3 ){ + newMax = sqlite3BtreeLastPage(pBt); + if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3; } + pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); + break; } +#endif -/* -** Allocate a Expr node which joins as many as two subtrees. + +#ifndef SQLITE_OMIT_TRACE +/* Opcode: Trace * * * P4 * ** -** One or both of the subtrees can be NULL. Return a pointer to the new -** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, -** free the subtrees and return NULL. +** If tracing is enabled (by the sqlite3_trace()) interface, then +** the UTF-8 string contained in P4 is emitted on the trace callback. */ -SQLITE_PRIVATE Expr *sqlite3PExpr( - Parse *pParse, /* Parsing context */ - int op, /* Expression opcode */ - Expr *pLeft, /* Left operand */ - Expr *pRight, /* Right operand */ - const Token *pToken /* Argument token */ -){ - Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); - sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); - if( p ) { - sqlite3ExprCheckHeight(pParse, p->nHeight); - } - return p; -} +case OP_Trace: { +#if 0 /* local variables moved into u.cn */ + char *zTrace; + char *z; +#endif /* local variables moved into u.cn */ -/* -** Join two expressions using an AND operator. If either expression is -** NULL, then just return the other expression. -*/ -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ - if( pLeft==0 ){ - return pRight; - }else if( pRight==0 ){ - return pLeft; - }else{ - Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); - sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); - return pNew; + if( db->xTrace && (u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ + u.cn.z = sqlite3VdbeExpandSql(p, u.cn.zTrace); + db->xTrace(db->pTraceArg, u.cn.z); + sqlite3DbFree(db, u.cn.z); + } +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace); } +#endif /* SQLITE_DEBUG */ + break; } +#endif + +/* Opcode: Noop * * * * * +** +** Do nothing. This instruction is often useful as a jump +** destination. +*/ /* -** Construct a new expression node for a function with multiple -** arguments. +** The magic Explain opcode are only inserted when explain==2 (which +** is to say when the EXPLAIN QUERY PLAN syntax is used.) +** This opcode records information from the optimizer. It is the +** the same as a no-op. This opcodesnever appears in a real VM program. */ -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ - Expr *pNew; - sqlite3 *db = pParse->db; - assert( pToken ); - pNew = sqlite3ExprAlloc(db, TK_FUNCTION, pToken, 1); - if( pNew==0 ){ - sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ - return 0; - } - pNew->x.pList = pList; - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - sqlite3ExprSetHeight(pParse, pNew); - return pNew; +default: { /* This is really OP_Noop and OP_Explain */ + assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); + break; } -/* -** Assign a variable number to an expression that encodes a wildcard -** in the original SQL statement. -** -** Wildcards consisting of a single "?" are assigned the next sequential -** variable number. -** -** Wildcards of the form "?nnn" are assigned the number "nnn". We make -** sure "nnn" is not too be to avoid a denial of service attack when -** the SQL statement comes from an external source. -** -** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number -** as the previous instance of the same wildcard. Or if this is the first -** instance of the wildcard, the next sequenial variable number is -** assigned. -*/ -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ - sqlite3 *db = pParse->db; - const char *z; +/***************************************************************************** +** The cases of the switch statement above this line should all be indented +** by 6 spaces. But the left-most 6 spaces have been removed to improve the +** readability. From this point on down, the normal indentation rules are +** restored. +*****************************************************************************/ + } - if( pExpr==0 ) return; - assert( !ExprHasAnyProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); - z = pExpr->u.zToken; - assert( z!=0 ); - assert( z[0]!=0 ); - if( z[1]==0 ){ - /* Wildcard of the form "?". Assign the next variable number */ - assert( z[0]=='?' ); - pExpr->iColumn = (ynVar)(++pParse->nVar); - }else if( z[0]=='?' ){ - /* Wildcard of the form "?nnn". Convert "nnn" to an integer and - ** use it as the variable number */ - i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); - pExpr->iColumn = (ynVar)i; - testcase( i==0 ); - testcase( i==1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ - sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", - db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; +#ifdef VDBE_PROFILE + { + u64 elapsed = sqlite3Hwtime() - start; + pOp->cycles += elapsed; + pOp->cnt++; +#if 0 + fprintf(stdout, "%10llu ", elapsed); + sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]); +#endif } - }else{ - /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable - ** number as the prior appearance of the same name, or if the name - ** has never appeared before, reuse the same variable number +#endif + + /* The following code adds nothing to the actual functionality + ** of the program. It is only here for testing and debugging. + ** On the other hand, it does burn CPU cycles every time through + ** the evaluator loop. So we can leave it out when NDEBUG is defined. */ - int i; - u32 n; - n = sqlite3Strlen30(z); - for(i=0; inVarExpr; i++){ - Expr *pE = pParse->apVarExpr[i]; - assert( pE!=0 ); - if( memcmp(pE->u.zToken, z, n)==0 && pE->u.zToken[n]==0 ){ - pExpr->iColumn = pE->iColumn; - break; - } - } - if( i>=pParse->nVarExpr ){ - pExpr->iColumn = (ynVar)(++pParse->nVar); - if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ - pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; - pParse->apVarExpr = - sqlite3DbReallocOrFree( - db, - pParse->apVarExpr, - pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) - ); +#ifndef NDEBUG + assert( pc>=-1 && pcnOp ); + +#ifdef SQLITE_DEBUG + if( p->trace ){ + if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc); + if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ + registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]); } - if( !db->mallocFailed ){ - assert( pParse->apVarExpr!=0 ); - pParse->apVarExpr[pParse->nVarExpr++] = pExpr; + if( pOp->opflags & OPFLG_OUT3 ){ + registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]); } } - } - if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ - sqlite3ErrorMsg(pParse, "too many SQL variables"); - } -} +#endif /* SQLITE_DEBUG */ +#endif /* NDEBUG */ + } /* The end of the for(;;) loop the loops through opcodes */ -/* -** Recursively delete an expression tree. -*/ -SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ - if( p==0 ) return; - if( !ExprHasAnyProperty(p, EP_TokenOnly) ){ - sqlite3ExprDelete(db, p->pLeft); - sqlite3ExprDelete(db, p->pRight); - if( !ExprHasProperty(p, EP_Reduced) && (p->flags2 & EP2_MallocedToken)!=0 ){ - sqlite3DbFree(db, p->u.zToken); - } - if( ExprHasProperty(p, EP_xIsSelect) ){ - sqlite3SelectDelete(db, p->x.pSelect); - }else{ - sqlite3ExprListDelete(db, p->x.pList); - } + /* If we reach this point, it means that execution is finished with + ** an error of some kind. + */ +vdbe_error_halt: + assert( rc ); + p->rc = rc; + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(rc, "statement aborts at %d: [%s] %s", + pc, p->zSql, p->zErrMsg); + sqlite3VdbeHalt(p); + if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; + rc = SQLITE_ERROR; + if( resetSchemaOnFault>0 ){ + sqlite3ResetInternalSchema(db, resetSchemaOnFault-1); } - if( !ExprHasProperty(p, EP_Static) ){ - sqlite3DbFree(db, p); + + /* This is the only way out of this procedure. We have to + ** release the mutexes on btrees that were acquired at the + ** top. */ +vdbe_return: + db->lastRowid = lastRowid; + sqlite3VdbeLeave(p); + return rc; + + /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH + ** is encountered. + */ +too_big: + sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); + rc = SQLITE_TOOBIG; + goto vdbe_error_halt; + + /* Jump to here if a malloc() fails. + */ +no_mem: + db->mallocFailed = 1; + sqlite3SetString(&p->zErrMsg, db, "out of memory"); + rc = SQLITE_NOMEM; + goto vdbe_error_halt; + + /* Jump to here for any other kind of fatal error. The "rc" variable + ** should hold the error number. + */ +abort_due_to_error: + assert( p->zErrMsg==0 ); + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc!=SQLITE_IOERR_NOMEM ){ + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); } -} + goto vdbe_error_halt; -/* -** Return the number of bytes allocated for the expression structure -** passed as the first argument. This is always one of EXPR_FULLSIZE, -** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. -*/ -static int exprStructSize(Expr *p){ - if( ExprHasProperty(p, EP_TokenOnly) ) return EXPR_TOKENONLYSIZE; - if( ExprHasProperty(p, EP_Reduced) ) return EXPR_REDUCEDSIZE; - return EXPR_FULLSIZE; + /* Jump to here if the sqlite3_interrupt() API sets the interrupt + ** flag. + */ +abort_due_to_interrupt: + assert( db->u1.isInterrupted ); + rc = SQLITE_INTERRUPT; + p->rc = rc; + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto vdbe_error_halt; } +/************** End of vdbe.c ************************************************/ +/************** Begin file vdbeblob.c ****************************************/ /* -** The dupedExpr*Size() routines each return the number of bytes required -** to store a copy of an expression or expression tree. They differ in -** how much of the tree is measured. -** -** dupedExprStructSize() Size of only the Expr structure -** dupedExprNodeSize() Size of Expr + space for token -** dupedExprSize() Expr + token + subtree components -** -*************************************************************************** +** 2007 May 1 ** -** The dupedExprStructSize() function returns two values OR-ed together: -** (1) the space required for a copy of the Expr structure only and -** (2) the EP_xxx flags that indicate what the structure size should be. -** The return values is always one of: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** EXPR_FULLSIZE -** EXPR_REDUCEDSIZE | EP_Reduced -** EXPR_TOKENONLYSIZE | EP_TokenOnly +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** The size of the structure can be found by masking the return value -** of this routine with 0xfff. The flags can be found by masking the -** return value with EP_Reduced|EP_TokenOnly. +************************************************************************* ** -** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size -** (unreduced) Expr objects as they or originally constructed by the parser. -** During expression analysis, extra information is computed and moved into -** later parts of teh Expr object and that extra information might get chopped -** off if the expression is reduced. Note also that it does not work to -** make a EXPRDUP_REDUCE copy of a reduced expression. It is only legal -** to reduce a pristine expression tree from the parser. The implementation -** of dupedExprStructSize() contain multiple assert() statements that attempt -** to enforce this constraint. +** This file contains code used to implement incremental BLOB I/O. */ -static int dupedExprStructSize(Expr *p, int flags){ - int nSize; - assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ - if( 0==(flags&EXPRDUP_REDUCE) ){ - nSize = EXPR_FULLSIZE; - }else{ - assert( !ExprHasAnyProperty(p, EP_TokenOnly|EP_Reduced) ); - assert( !ExprHasProperty(p, EP_FromJoin) ); - assert( (p->flags2 & EP2_MallocedToken)==0 ); - assert( (p->flags2 & EP2_Irreducible)==0 ); - if( p->pLeft || p->pRight || p->pColl || p->x.pList ){ - nSize = EXPR_REDUCEDSIZE | EP_Reduced; - }else{ - nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; - } - } - return nSize; -} + + +#ifndef SQLITE_OMIT_INCRBLOB /* -** This function returns the space in bytes required to store the copy -** of the Expr structure and a copy of the Expr.u.zToken string (if that -** string is defined.) +** Valid sqlite3_blob* handles point to Incrblob structures. */ -static int dupedExprNodeSize(Expr *p, int flags){ - int nByte = dupedExprStructSize(p, flags) & 0xfff; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nByte += sqlite3Strlen30(p->u.zToken)+1; - } - return ROUND8(nByte); -} +typedef struct Incrblob Incrblob; +struct Incrblob { + int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ + int nByte; /* Size of open blob, in bytes */ + int iOffset; /* Byte offset of blob in cursor data */ + int iCol; /* Table column this handle is open on */ + BtCursor *pCsr; /* Cursor pointing at blob row */ + sqlite3_stmt *pStmt; /* Statement holding cursor open */ + sqlite3 *db; /* The associated database */ +}; + /* -** Return the number of bytes required to create a duplicate of the -** expression passed as the first argument. The second argument is a -** mask containing EXPRDUP_XXX flags. +** This function is used by both blob_open() and blob_reopen(). It seeks +** the b-tree cursor associated with blob handle p to point to row iRow. +** If successful, SQLITE_OK is returned and subsequent calls to +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. ** -** The value returned includes space to create a copy of the Expr struct -** itself and the buffer referred to by Expr.u.zToken, if any. +** If an error occurs, or if the specified row does not exist or does not +** contain a value of type TEXT or BLOB in the column nominated when the +** blob handle was opened, then an error code is returned and *pzErr may +** be set to point to a buffer containing an error message. It is the +** responsibility of the caller to free the error message buffer using +** sqlite3DbFree(). ** -** If the EXPRDUP_REDUCE flag is set, then the return value includes -** space to duplicate all Expr nodes in the tree formed by Expr.pLeft -** and Expr.pRight variables (but not for any structures pointed to or -** descended from the Expr.x.pList or Expr.x.pSelect variables). +** If an error does occur, then the b-tree cursor is closed. All subsequent +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will +** immediately return SQLITE_ABORT. */ -static int dupedExprSize(Expr *p, int flags){ - int nByte = 0; - if( p ){ - nByte = dupedExprNodeSize(p, flags); - if( flags&EXPRDUP_REDUCE ){ - nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ + int rc; /* Error code */ + char *zErr = 0; /* Error message */ + Vdbe *v = (Vdbe *)p->pStmt; + + /* Set the value of the SQL statements only variable to integer iRow. + ** This is done directly instead of using sqlite3_bind_int64() to avoid + ** triggering asserts related to mutexes. + */ + assert( v->aVar[0].flags&MEM_Int ); + v->aVar[0].u.i = iRow; + + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_ROW ){ + u32 type = v->apCsr[0]->aType[p->iCol]; + if( type<12 ){ + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", + type==0?"null": type==7?"real": "integer" + ); + rc = SQLITE_ERROR; + sqlite3_finalize(p->pStmt); + p->pStmt = 0; + }else{ + p->iOffset = v->apCsr[0]->aOffset[p->iCol]; + p->nByte = sqlite3VdbeSerialTypeLen(type); + p->pCsr = v->apCsr[0]->pCursor; + sqlite3BtreeEnterCursor(p->pCsr); + sqlite3BtreeCacheOverflow(p->pCsr); + sqlite3BtreeLeaveCursor(p->pCsr); } } - return nByte; + + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( p->pStmt ){ + rc = sqlite3_finalize(p->pStmt); + p->pStmt = 0; + if( rc==SQLITE_OK ){ + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); + rc = SQLITE_ERROR; + }else{ + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); + } + } + + assert( rc!=SQLITE_OK || zErr==0 ); + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); + + *pzErr = zErr; + return rc; } /* -** This function is similar to sqlite3ExprDup(), except that if pzBuffer -** is not NULL then *pzBuffer is assumed to point to a buffer large enough -** to store the copy of expression p, the copies of p->u.zToken -** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte passed the -** portion of the buffer copied into by this function. +** Open a blob handle. */ -static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ - Expr *pNew = 0; /* Value to return */ - if( p ){ - const int isReduced = (flags&EXPRDUP_REDUCE); - u8 *zAlloc; - u32 staticFlag = 0; +SQLITE_API int sqlite3_blob_open( + sqlite3* db, /* The database connection */ + const char *zDb, /* The attached database containing the blob */ + const char *zTable, /* The table containing the blob */ + const char *zColumn, /* The column containing the blob */ + sqlite_int64 iRow, /* The row containing the glob */ + int flags, /* True -> read/write access, false -> read-only */ + sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ +){ + int nAttempt = 0; + int iCol; /* Index of zColumn in row-record */ - assert( pzBuffer==0 || isReduced ); + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const VdbeOpList openBlob[] = { + {OP_Transaction, 0, 0, 0}, /* 0: Start a transaction */ + {OP_VerifyCookie, 0, 0, 0}, /* 1: Check the schema cookie */ + {OP_TableLock, 0, 0, 0}, /* 2: Acquire a read or write lock */ - /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; - staticFlag = EP_Static; - }else{ - zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); - } - pNew = (Expr *)zAlloc; + /* One of the following two instructions is replaced by an OP_Noop. */ + {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ + {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ - if( pNew ){ - /* Set nNewSize to the size allocated for the structure pointed to - ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or - ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed - ** by the copy of the p->u.zToken string (if any). - */ - const unsigned nStructSize = dupedExprStructSize(p, flags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; - }else{ - nToken = 0; - } - if( isReduced ){ - assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); - }else{ - int nSize = exprStructSize(p); - memcpy(zAlloc, p, nSize); - memset(&zAlloc[nSize], 0, EXPR_FULLSIZE-nSize); - } + {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ + {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 7 */ + {OP_ResultRow, 1, 0, 0}, /* 8 */ + {OP_Goto, 0, 5, 0}, /* 9 */ + {OP_Close, 0, 0, 0}, /* 10 */ + {OP_Halt, 0, 0, 0}, /* 11 */ + }; - /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); - pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); - pNew->flags |= staticFlag; + int rc = SQLITE_OK; + char *zErr = 0; + Table *pTab; + Parse *pParse = 0; + Incrblob *pBlob = 0; - /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; - memcpy(zToken, p->u.zToken, nToken); + flags = !!flags; /* flags = (flags ? 1 : 0); */ + *ppBlob = 0; + + sqlite3_mutex_enter(db->mutex); + + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); + if( !pBlob ) goto blob_open_out; + pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); + if( !pParse ) goto blob_open_out; + + do { + memset(pParse, 0, sizeof(Parse)); + pParse->db = db; + sqlite3DbFree(db, zErr); + zErr = 0; + + sqlite3BtreeEnterAll(db); + pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); + if( pTab && IsVirtual(pTab) ){ + pTab = 0; + sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); + } +#ifndef SQLITE_OMIT_VIEW + if( pTab && pTab->pSelect ){ + pTab = 0; + sqlite3ErrorMsg(pParse, "cannot open view: %s", zTable); + } +#endif + if( !pTab ){ + if( pParse->zErrMsg ){ + sqlite3DbFree(db, zErr); + zErr = pParse->zErrMsg; + pParse->zErrMsg = 0; } + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } - if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ - /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ - if( ExprHasProperty(p, EP_xIsSelect) ){ - pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); - }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); - } + /* Now search pTab for the exact column. */ + for(iCol=0; iColnCol; iCol++) { + if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ + break; } + } + if( iCol==pTab->nCol ){ + sqlite3DbFree(db, zErr); + zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasAnyProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, flags); - if( ExprHasProperty(pNew, EP_Reduced) ){ - pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); - pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); - } - if( pzBuffer ){ - *pzBuffer = zAlloc; + /* If the value is being opened for writing, check that the + ** column is not indexed, and that it is not part of a foreign key. + ** It is against the rules to open a column to which either of these + ** descriptions applies for writing. */ + if( flags ){ + const char *zFault = 0; + Index *pIdx; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( db->flags&SQLITE_ForeignKeys ){ + /* Check that the column is not part of an FK child key definition. It + ** is not necessary to check if it is part of a parent key, as parent + ** key columns must be indexed. The check below will pick up this + ** case. */ + FKey *pFKey; + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + int j; + for(j=0; jnCol; j++){ + if( pFKey->aCol[j].iFrom==iCol ){ + zFault = "foreign key"; + } + } } - }else{ - pNew->flags2 = 0; - if( !ExprHasAnyProperty(p, EP_TokenOnly) ){ - pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); - pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); + } +#endif + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int j; + for(j=0; jnColumn; j++){ + if( pIdx->aiColumn[j]==iCol ){ + zFault = "indexed"; + } } } - + if( zFault ){ + sqlite3DbFree(db, zErr); + zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } } - } - return pNew; -} -/* -** The following group of routines make deep copies of expressions, -** expression lists, ID lists, and select statements. The copies can -** be deleted (by being passed to their respective ...Delete() routines) -** without effecting the originals. -** -** The expression list, ID, and source lists return by sqlite3ExprListDup(), -** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded -** by subsequent calls to sqlite*ListAppend() routines. -** -** Any tables that the SrcList might point to are not duplicated. -** -** The flags parameter contains a combination of the EXPRDUP_XXX flags. -** If the EXPRDUP_REDUCE flag is set, then the structure returned is a -** truncated version of the usual Expr structure that will be stored as -** part of the in-memory representation of the database schema. -*/ -SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ - return exprDup(db, p, flags, 0); -} -SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ - ExprList *pNew; - struct ExprList_item *pItem, *pOldItem; - int i; - if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); - if( pNew==0 ) return 0; - pNew->iECursor = 0; - pNew->nExpr = pNew->nAlloc = p->nExpr; - pNew->a = pItem = sqlite3DbMallocRaw(db, p->nExpr*sizeof(p->a[0]) ); - if( pItem==0 ){ - sqlite3DbFree(db, pNew); - return 0; - } - pOldItem = p->a; - for(i=0; inExpr; i++, pItem++, pOldItem++){ - Expr *pOldExpr = pOldItem->pExpr; - pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); - pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); - pItem->sortOrder = pOldItem->sortOrder; - pItem->done = 0; - pItem->iCol = pOldItem->iCol; - pItem->iAlias = pOldItem->iAlias; - } - return pNew; -} + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db); + assert( pBlob->pStmt || db->mallocFailed ); + if( pBlob->pStmt ){ + Vdbe *v = (Vdbe *)pBlob->pStmt; + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); -/* -** If cursors, triggers, views and subqueries are all omitted from -** the build, then none of the following routines, except for -** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes -** called with a NULL argument. -*/ -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ - || !defined(SQLITE_OMIT_SUBQUERY) -SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ - SrcList *pNew; - int i; - int nByte; - if( p==0 ) return 0; - nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRaw(db, nByte ); - if( pNew==0 ) return 0; - pNew->nSrc = pNew->nAlloc = p->nSrc; - for(i=0; inSrc; i++){ - struct SrcList_item *pNewItem = &pNew->a[i]; - struct SrcList_item *pOldItem = &p->a[i]; - Table *pTab; - pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); - pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; - pNewItem->iCursor = pOldItem->iCursor; - pNewItem->isPopulated = pOldItem->isPopulated; - pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); - pNewItem->notIndexed = pOldItem->notIndexed; - pNewItem->pIndex = pOldItem->pIndex; - pTab = pNewItem->pTab = pOldItem->pTab; - if( pTab ){ - pTab->nRef++; - } - pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); - pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); - pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); - pNewItem->colUsed = pOldItem->colUsed; - } - return pNew; -} -SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ - IdList *pNew; - int i; - if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); - if( pNew==0 ) return 0; - pNew->nId = pNew->nAlloc = p->nId; - pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); - if( pNew->a==0 ){ - sqlite3DbFree(db, pNew); - return 0; - } - for(i=0; inId; i++){ - struct IdList_item *pNewItem = &pNew->a[i]; - struct IdList_item *pOldItem = &p->a[i]; - pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pNewItem->idx = pOldItem->idx; - } - return pNew; -} -SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - Select *pNew; - if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); - if( pNew==0 ) return 0; - pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); - pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); - pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); - pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); - pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); - pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); - pNew->op = p->op; - pNew->pPrior = sqlite3SelectDup(db, p->pPrior, flags); - pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); - pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); - pNew->iLimit = 0; - pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; - pNew->pRightmost = 0; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; - return pNew; -} + sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); + + + /* Configure the OP_Transaction */ + sqlite3VdbeChangeP1(v, 0, iDb); + sqlite3VdbeChangeP2(v, 0, flags); + + /* Configure the OP_VerifyCookie */ + sqlite3VdbeChangeP1(v, 1, iDb); + sqlite3VdbeChangeP2(v, 1, pTab->pSchema->schema_cookie); + sqlite3VdbeChangeP3(v, 1, pTab->pSchema->iGeneration); + + /* Make sure a mutex is held on the table to be accessed */ + sqlite3VdbeUsesBtree(v, iDb); + + /* Configure the OP_TableLock instruction */ +#ifdef SQLITE_OMIT_SHARED_CACHE + sqlite3VdbeChangeToNoop(v, 2, 1); #else -SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - assert( p==0 ); - return 0; -} + sqlite3VdbeChangeP1(v, 2, iDb); + sqlite3VdbeChangeP2(v, 2, pTab->tnum); + sqlite3VdbeChangeP3(v, 2, flags); + sqlite3VdbeChangeP4(v, 2, pTab->zName, P4_TRANSIENT); #endif + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + sqlite3VdbeChangeToNoop(v, 4 - flags, 1); + sqlite3VdbeChangeP2(v, 3 + flags, pTab->tnum); + sqlite3VdbeChangeP3(v, 3 + flags, iDb); -/* -** Add a new element to the end of an expression list. If pList is -** initially NULL, then create a new expression list. -** -** If a memory allocation error occurs, the entire list is freed and -** NULL is returned. If non-NULL is returned, then it is guaranteed -** that the new entry was successfully appended. -*/ -SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* List to which to append. Might be NULL */ - Expr *pExpr /* Expression to be appended. Might be NULL */ -){ - sqlite3 *db = pParse->db; - if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); - if( pList==0 ){ - goto no_mem; + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); + sqlite3VdbeChangeP2(v, 7, pTab->nCol); + if( !db->mallocFailed ){ + pParse->nVar = 1; + pParse->nMem = 1; + pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); + } } - assert( pList->nAlloc==0 ); - } - if( pList->nAlloc<=pList->nExpr ){ - struct ExprList_item *a; - int n = pList->nAlloc*2 + 4; - a = sqlite3DbRealloc(db, pList->a, n*sizeof(pList->a[0])); - if( a==0 ){ - goto no_mem; + + pBlob->flags = flags; + pBlob->iCol = iCol; + pBlob->db = db; + sqlite3BtreeLeaveAll(db); + if( db->mallocFailed ){ + goto blob_open_out; } - pList->a = a; - pList->nAlloc = sqlite3DbMallocSize(db, a)/sizeof(a[0]); - } - assert( pList->a!=0 ); - if( 1 ){ - struct ExprList_item *pItem = &pList->a[pList->nExpr++]; - memset(pItem, 0, sizeof(*pItem)); - pItem->pExpr = pExpr; - } - return pList; + sqlite3_bind_int64(pBlob->pStmt, 1, iRow); + rc = blobSeekToRow(pBlob, iRow, &zErr); + } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); -no_mem: - /* Avoid leaking memory if malloc has failed. */ - sqlite3ExprDelete(db, pExpr); - sqlite3ExprListDelete(db, pList); - return 0; +blob_open_out: + if( rc==SQLITE_OK && db->mallocFailed==0 ){ + *ppBlob = (sqlite3_blob *)pBlob; + }else{ + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); + sqlite3DbFree(db, pBlob); + } + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + sqlite3StackFree(db, pParse); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* -** Set the ExprList.a[].zName element of the most recently added item -** on the expression list. -** -** pList might be NULL following an OOM error. But pName should never be -** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag -** is set. +** Close a blob handle that was previously created using +** sqlite3_blob_open(). */ -SQLITE_PRIVATE void sqlite3ExprListSetName( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* List to which to add the span. */ - Token *pName, /* Name to be added */ - int dequote /* True to cause the name to be dequoted */ -){ - assert( pList!=0 || pParse->db->mallocFailed!=0 ); - if( pList ){ - struct ExprList_item *pItem; - assert( pList->nExpr>0 ); - pItem = &pList->a[pList->nExpr-1]; - assert( pItem->zName==0 ); - pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); +SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ + Incrblob *p = (Incrblob *)pBlob; + int rc; + sqlite3 *db; + + if( p ){ + db = p->db; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3_finalize(p->pStmt); + sqlite3DbFree(db, p); + sqlite3_mutex_leave(db->mutex); + }else{ + rc = SQLITE_OK; } + return rc; } /* -** Set the ExprList.a[].zSpan element of the most recently added item -** on the expression list. -** -** pList might be NULL following an OOM error. But pSpan should never be -** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag -** is set. +** Perform a read or write operation on a blob */ -SQLITE_PRIVATE void sqlite3ExprListSetSpan( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* List to which to add the span. */ - ExprSpan *pSpan /* The span to be added */ +static int blobReadWrite( + sqlite3_blob *pBlob, + void *z, + int n, + int iOffset, + int (*xCall)(BtCursor*, u32, u32, void*) ){ - sqlite3 *db = pParse->db; - assert( pList!=0 || db->mallocFailed!=0 ); - if( pList ){ - struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; - assert( pList->nExpr>0 ); - assert( db->mallocFailed || pItem->pExpr==pSpan->pExpr ); - sqlite3DbFree(db, pItem->zSpan); - pItem->zSpan = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); + int rc; + Incrblob *p = (Incrblob *)pBlob; + Vdbe *v; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + v = (Vdbe*)p->pStmt; + + if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ + /* Request is out of range. Return a transient error. */ + rc = SQLITE_ERROR; + sqlite3Error(db, SQLITE_ERROR, 0); + }else if( v==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is + ** returned, clean-up the statement handle. + */ + assert( db == v->db ); + sqlite3BtreeEnterCursor(p->pCsr); + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); + sqlite3BtreeLeaveCursor(p->pCsr); + if( rc==SQLITE_ABORT ){ + sqlite3VdbeFinalize(v); + p->pStmt = 0; + }else{ + db->errCode = rc; + v->rc = rc; + } } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* -** If the expression list pEList contains more than iLimit elements, -** leave an error message in pParse. +** Read data from a blob handle. */ -SQLITE_PRIVATE void sqlite3ExprListCheckLength( - Parse *pParse, - ExprList *pEList, - const char *zObject -){ - int mx = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; - testcase( pEList && pEList->nExpr==mx ); - testcase( pEList && pEList->nExpr==mx+1 ); - if( pEList && pEList->nExpr>mx ){ - sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); - } +SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ + return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData); } /* -** Delete an entire expression list. +** Write data to a blob handle. */ -SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ - int i; - struct ExprList_item *pItem; - if( pList==0 ) return; - assert( pList->a!=0 || (pList->nExpr==0 && pList->nAlloc==0) ); - assert( pList->nExpr<=pList->nAlloc ); - for(pItem=pList->a, i=0; inExpr; i++, pItem++){ - sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zName); - sqlite3DbFree(db, pItem->zSpan); - } - sqlite3DbFree(db, pList->a); - sqlite3DbFree(db, pList); +SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ + return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.pi to 0 if the expression is -** not constant. -** -** These callback routines are used to implement the following: +** Query a blob handle for the size of the data. ** -** sqlite3ExprIsConstant() -** sqlite3ExprIsConstantNotJoin() -** sqlite3ExprIsConstantOrFunction() +** The Incrblob.nByte field is fixed for the lifetime of the Incrblob +** so no mutex is required for access. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ + Incrblob *p = (Incrblob *)pBlob; + return (p && p->pStmt) ? p->nByte : 0; +} + +/* +** Move an existing blob handle to point to a different row of the same +** database table. ** +** If an error occurs, or if the specified row does not exist or does not +** contain a blob or text value, then an error code is returned and the +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** immediately return SQLITE_ABORT. */ -static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + sqlite3 *db; - /* If pWalker->u.i is 3 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression - ** from being considered constant. */ - if( pWalker->u.i==3 && ExprHasAnyProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; - return WRC_Abort; - } + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); - switch( pExpr->op ){ - /* Consider functions to be constant if all their arguments are constant - ** and pWalker->u.i==2 */ - case TK_FUNCTION: - if( pWalker->u.i==2 ) return 0; - /* Fall through */ - case TK_ID: - case TK_COLUMN: - case TK_AGG_FUNCTION: - case TK_AGG_COLUMN: - testcase( pExpr->op==TK_ID ); - testcase( pExpr->op==TK_COLUMN ); - testcase( pExpr->op==TK_AGG_FUNCTION ); - testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; - default: - testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ - testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ - return WRC_Continue; + if( p->pStmt==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + char *zErr; + rc = blobSeekToRow(p, iRow, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + } + assert( rc!=SQLITE_SCHEMA ); } + + rc = sqlite3ApiExit(db, rc); + assert( rc==SQLITE_OK || p->pStmt==0 ); + sqlite3_mutex_leave(db->mutex); + return rc; } -static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ - UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; - return WRC_Abort; -} -static int exprIsConst(Expr *p, int initFlag){ - Walker w; - w.u.i = initFlag; - w.xExprCallback = exprNodeIsConstant; - w.xSelectCallback = selectNodeIsConstant; - sqlite3WalkExpr(&w, p); - return w.u.i; -} +#endif /* #ifndef SQLITE_OMIT_INCRBLOB */ + +/************** End of vdbeblob.c ********************************************/ +/************** Begin file journal.c *****************************************/ /* -** Walk an expression tree. Return 1 if the expression is constant -** and 0 if it involves variables or function calls. +** 2007 August 22 ** -** For the purposes of this function, a double-quoted string (ex: "abc") -** is considered a variable but a single-quoted string (ex: 'abc') is -** a constant. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a special kind of sqlite3_file object used +** by SQLite to create journal files if the atomic-write optimization +** is enabled. +** +** The distinctive characteristic of this sqlite3_file is that the +** actual on disk file is created lazily. When the file is created, +** the caller specifies a buffer size for an in-memory buffer to +** be used to service read() and write() requests. The actual file +** on disk is not created or populated until either: +** +** 1) The in-memory representation grows too large for the allocated +** buffer, or +** 2) The sqlite3JournalCreate() function is called. */ -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); -} +#ifdef SQLITE_ENABLE_ATOMIC_WRITE -/* -** Walk an expression tree. Return 1 if the expression is constant -** that does no originate from the ON or USING clauses of a join. -** Return 0 if it involves variables or function calls or terms from -** an ON or USING clause. -*/ -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 3); -} /* -** Walk an expression tree. Return 1 if the expression is constant -** or a function call with constant arguments. Return and 0 if there -** are any variables. -** -** For the purposes of this function, a double-quoted string (ex: "abc") -** is considered a variable but a single-quoted string (ex: 'abc') is -** a constant. +** A JournalFile object is a subclass of sqlite3_file used by +** as an open file handle for journal files. */ -SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p){ - return exprIsConst(p, 2); -} +struct JournalFile { + sqlite3_io_methods *pMethod; /* I/O methods on journal files */ + int nBuf; /* Size of zBuf[] in bytes */ + char *zBuf; /* Space to buffer journal writes */ + int iSize; /* Amount of zBuf[] currently used */ + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + sqlite3_file *pReal; /* The "real" underlying file descriptor */ + const char *zJournal; /* Name of the journal file */ +}; +typedef struct JournalFile JournalFile; /* -** If the expression p codes a constant integer that is small enough -** to fit in a 32-bit integer, return 1 and put the value of the integer -** in *pValue. If the expression is not an integer or if it is too big -** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** If it does not already exists, create and populate the on-disk file +** for JournalFile p. */ -SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ - int rc = 0; - if( p->flags & EP_IntValue ){ - *pValue = p->u.iValue; - return 1; - } - switch( p->op ){ - case TK_INTEGER: { - rc = sqlite3GetInt32(p->u.zToken, pValue); - assert( rc==0 ); - break; - } - case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue); - break; - } - case TK_UMINUS: { - int v; - if( sqlite3ExprIsInteger(p->pLeft, &v) ){ - *pValue = -v; - rc = 1; +static int createFile(JournalFile *p){ + int rc = SQLITE_OK; + if( !p->pReal ){ + sqlite3_file *pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); + if( rc==SQLITE_OK ){ + p->pReal = pReal; + if( p->iSize>0 ){ + assert(p->iSize<=p->nBuf); + rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); } - break; } - default: break; - } - if( rc ){ - assert( ExprHasAnyProperty(p, EP_Reduced|EP_TokenOnly) - || (p->flags2 & EP2_MallocedToken)==0 ); - p->op = TK_INTEGER; - p->flags |= EP_IntValue; - p->u.iValue = *pValue; } return rc; } /* -** Return FALSE if there is no chance that the expression can be NULL. -** -** If the expression might be NULL or if the expression is too complex -** to tell return TRUE. -** -** This routine is used as an optimization, to skip OP_IsNull opcodes -** when we know that a value cannot be NULL. Hence, a false positive -** (returning TRUE when in fact the expression can never be NULL) might -** be a small performance hit but is otherwise harmless. On the other -** hand, a false negative (returning FALSE when the result could be NULL) -** will likely result in an incorrect answer. So when in doubt, return -** TRUE. +** Close the file. */ -SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ - u8 op; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } - op = p->op; - if( op==TK_REGISTER ) op = p->op2; - switch( op ){ - case TK_INTEGER: - case TK_STRING: - case TK_FLOAT: - case TK_BLOB: - return 0; - default: - return 1; +static int jrnlClose(sqlite3_file *pJfd){ + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + sqlite3OsClose(p->pReal); } + sqlite3_free(p->zBuf); + return SQLITE_OK; } /* -** Generate an OP_IsNull instruction that tests register iReg and jumps -** to location iDest if the value in iReg is NULL. The value in iReg -** was computed by pExpr. If we can look at pExpr at compile-time and -** determine that it can never generate a NULL, then the OP_IsNull operation -** can be omitted. +** Read data from the file. */ -SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump( - Vdbe *v, /* The VDBE under construction */ - const Expr *pExpr, /* Only generate OP_IsNull if this expr can be NULL */ - int iReg, /* Test the value in this register for NULL */ - int iDest /* Jump here if the value is null */ +static int jrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ ){ - if( sqlite3ExprCanBeNull(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest); + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); + }else if( (iAmt+iOfst)>p->iSize ){ + rc = SQLITE_IOERR_SHORT_READ; + }else{ + memcpy(zBuf, &p->zBuf[iOfst], iAmt); } + return rc; } /* -** Return TRUE if the given expression is a constant which would be -** unchanged by OP_Affinity with the affinity given in the second -** argument. -** -** This routine is used to determine if the OP_Affinity operation -** can be omitted. When in doubt return FALSE. A false negative -** is harmless. A false positive, however, can result in the wrong -** answer. +** Write data to the file. */ -SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ - u8 op; - if( aff==SQLITE_AFF_NONE ) return 1; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } - op = p->op; - if( op==TK_REGISTER ) op = p->op2; - switch( op ){ - case TK_INTEGER: { - return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; - } - case TK_FLOAT: { - return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; - } - case TK_STRING: { - return aff==SQLITE_AFF_TEXT; - } - case TK_BLOB: { - return 1; - } - case TK_COLUMN: { - assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ - return p->iColumn<0 - && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); - } - default: { - return 0; +static int jrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ + rc = createFile(p); + } + if( rc==SQLITE_OK ){ + if( p->pReal ){ + rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); + }else{ + memcpy(&p->zBuf[iOfst], zBuf, iAmt); + if( p->iSize<(iOfst+iAmt) ){ + p->iSize = (iOfst+iAmt); + } } } + return rc; } /* -** Return TRUE if the given string is a row-id column name. +** Truncate the file. */ -SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ - if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1; - if( sqlite3StrICmp(z, "ROWID")==0 ) return 1; - if( sqlite3StrICmp(z, "OID")==0 ) return 1; - return 0; +static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsTruncate(p->pReal, size); + }else if( sizeiSize ){ + p->iSize = size; + } + return rc; } /* -** Return true if we are able to the IN operator optimization on a -** query of the form -** -** x IN (SELECT ...) -** -** Where the SELECT... clause is as specified by the parameter to this -** routine. -** -** The Select object passed in has already been preprocessed and no -** errors have been found. +** Sync the file. */ -#ifndef SQLITE_OMIT_SUBQUERY -static int isCandidateForInOpt(Select *p){ - SrcList *pSrc; - ExprList *pEList; - Table *pTab; - if( p==0 ) return 0; /* right-hand side of IN is SELECT */ - if( p->pPrior ) return 0; /* Not a compound SELECT */ - if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ - testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); - testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); - return 0; /* No DISTINCT keyword and no aggregate functions */ +static int jrnlSync(sqlite3_file *pJfd, int flags){ + int rc; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsSync(p->pReal, flags); + }else{ + rc = SQLITE_OK; } - assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ - if( p->pLimit ) return 0; /* Has no LIMIT clause */ - assert( p->pOffset==0 ); /* No LIMIT means no OFFSET */ - if( p->pWhere ) return 0; /* Has no WHERE clause */ - pSrc = p->pSrc; - assert( pSrc!=0 ); - if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ - pTab = pSrc->a[0].pTab; - if( NEVER(pTab==0) ) return 0; - assert( pTab->pSelect==0 ); /* FROM clause is not a view */ - if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ - pEList = p->pEList; - if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ - if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ - return 1; + return rc; } -#endif /* SQLITE_OMIT_SUBQUERY */ /* -** This function is used by the implementation of the IN (...) operator. -** It's job is to find or create a b-tree structure that may be used -** either to test for membership of the (...) set or to iterate through -** its members, skipping duplicates. -** -** The index of the cursor opened on the b-tree (database table, database index -** or ephermal table) is stored in pX->iTable before this function returns. -** The returned value of this function indicates the b-tree type, as follows: -** -** IN_INDEX_ROWID - The cursor was opened on a database table. -** IN_INDEX_INDEX - The cursor was opened on a database index. -** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. -** -** An existing b-tree may only be used if the SELECT is of the simple -** form: -** -** SELECT FROM
    • -** -** If the prNotFound parameter is 0, then the b-tree will be used to iterate -** through the set members, skipping any duplicates. In this case an -** epheremal table must be used unless the selected is guaranteed -** to be unique - either because it is an INTEGER PRIMARY KEY or it -** has a UNIQUE constraint or UNIQUE index. -** -** If the prNotFound parameter is not 0, then the b-tree will be used -** for fast set membership tests. In this case an epheremal table must -** be used unless is an INTEGER PRIMARY KEY or an index can -** be found with as its left-most column. -** -** When the b-tree is being used for membership tests, the calling function -** needs to know whether or not the structure contains an SQL NULL -** value in order to correctly evaluate expressions like "X IN (Y, Z)". -** If there is any chance that the (...) might contain a NULL value at -** runtime, then a register is allocated and the register number written -** to *prNotFound. If there is no chance that the (...) contains a -** NULL value, then *prNotFound is left unchanged. -** -** If a register is allocated and its location stored in *prNotFound, then -** its initial value is NULL. If the (...) does not remain constant -** for the duration of the query (i.e. the SELECT within the (...) -** is a correlated subquery) then the value of the allocated register is -** reset to NULL each time the subquery is rerun. This allows the -** caller to use vdbe code equivalent to the following: -** -** if( register==NULL ){ -** has_null = -** register = 1 -** } -** -** in order to avoid running the -** test more often than is necessary. +** Query the size of the file in bytes. */ -#ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ - Select *p; /* SELECT to the right of IN operator */ - int eType = 0; /* Type of RHS table. IN_INDEX_* */ - int iTab = pParse->nTab++; /* Cursor of the RHS table */ - int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ - - assert( pX->op==TK_IN ); - - /* Check to see if an existing table or index can be used to - ** satisfy the query. This is preferable to generating a new - ** ephemeral table. - */ - p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); - if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ - sqlite3 *db = pParse->db; /* Database connection */ - Expr *pExpr = p->pEList->a[0].pExpr; /* Expression */ - int iCol = pExpr->iColumn; /* Index of column */ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - Table *pTab = p->pSrc->a[0].pTab; /* Table
    . */ - int iDb; /* Database idx for pTab */ - - /* Code an OP_VerifyCookie and OP_TableLock for
    . */ - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - sqlite3CodeVerifySchema(pParse, iDb); - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - - /* This function is only called from two places. In both cases the vdbe - ** has already been allocated. So assume sqlite3GetVdbe() is always - ** successful here. - */ - assert(v); - if( iCol<0 ){ - int iMem = ++pParse->nMem; - int iAddr; - - iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); - sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); - - sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); - eType = IN_INDEX_ROWID; - - sqlite3VdbeJumpHere(v, iAddr); - }else{ - Index *pIdx; /* Iterator variable */ - - /* The collation sequence used by the comparison. If an index is to - ** be used in place of a temp-table, it must be ordered according - ** to this collation sequence. */ - CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); - - /* Check that the affinity that will be used to perform the - ** comparison is the same as the affinity of the column. If - ** it is not, it is not possible to use any index. - */ - char aff = comparisonAffinity(pX); - int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); +static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsFileSize(p->pReal, pSize); + }else{ + *pSize = (sqlite_int64) p->iSize; + } + return rc; +} - for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ - if( (pIdx->aiColumn[0]==iCol) - && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nColumn==1 && pIdx->onError!=OE_None)) - ){ - int iMem = ++pParse->nMem; - int iAddr; - char *pKey; - - pKey = (char *)sqlite3IndexKeyinfo(pParse, pIdx); - iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); - sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); - - sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, - pKey,P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIdx->zName)); - eType = IN_INDEX_INDEX; +/* +** Table of methods for JournalFile sqlite3_file object. +*/ +static struct sqlite3_io_methods JournalFileMethods = { + 1, /* iVersion */ + jrnlClose, /* xClose */ + jrnlRead, /* xRead */ + jrnlWrite, /* xWrite */ + jrnlTruncate, /* xTruncate */ + jrnlSync, /* xSync */ + jrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0 /* xShmUnmap */ +}; - sqlite3VdbeJumpHere(v, iAddr); - if( prNotFound && !pTab->aCol[iCol].notNull ){ - *prNotFound = ++pParse->nMem; - } - } - } +/* +** Open a journal file. +*/ +SQLITE_PRIVATE int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nBuf /* Bytes buffered before opening the file */ +){ + JournalFile *p = (JournalFile *)pJfd; + memset(p, 0, sqlite3JournalSize(pVfs)); + if( nBuf>0 ){ + p->zBuf = sqlite3MallocZero(nBuf); + if( !p->zBuf ){ + return SQLITE_NOMEM; } + }else{ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); } + p->pMethod = &JournalFileMethods; + p->nBuf = nBuf; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} - if( eType==0 ){ - /* Could not found an existing table or index to use as the RHS b-tree. - ** We will have to generate an ephemeral table to do the job. - */ - double savedNQueryLoop = pParse->nQueryLoop; - int rMayHaveNull = 0; - eType = IN_INDEX_EPH; - if( prNotFound ){ - *prNotFound = rMayHaveNull = ++pParse->nMem; - }else{ - testcase( pParse->nQueryLoop>(double)1 ); - pParse->nQueryLoop = (double)1; - if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ - eType = IN_INDEX_ROWID; - } - } - sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); - pParse->nQueryLoop = savedNQueryLoop; - }else{ - pX->iTable = iTab; +/* +** If the argument p points to a JournalFile structure, and the underlying +** file has not yet been created, create it now. +*/ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ + if( p->pMethods!=&JournalFileMethods ){ + return SQLITE_OK; } - return eType; + return createFile((JournalFile *)p); +} + +/* +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. +*/ +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return (pVfs->szOsFile+sizeof(JournalFile)); } #endif +/************** End of journal.c *********************************************/ +/************** Begin file memjournal.c **************************************/ /* -** Generate code for scalar subqueries used as a subquery expression, EXISTS, -** or IN operators. Examples: -** -** (SELECT a FROM b) -- subquery -** EXISTS (SELECT a FROM b) -- EXISTS subquery -** x IN (4,5,11) -- IN operator with list on right-hand side -** x IN (SELECT a FROM b) -- IN operator with subquery on the right -** -** The pExpr parameter describes the expression that contains the IN -** operator or subquery. +** 2008 October 7 ** -** If parameter isRowid is non-zero, then expression pExpr is guaranteed -** to be of the form " IN (?, ?, ?)", where is a reference -** to some integer key column of a table B-Tree. In this case, use an -** intkey B-Tree to store the set of IN(...) values instead of the usual -** (slower) variable length keys B-Tree. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** If rMayHaveNull is non-zero, that means that the operation is an IN -** (not a SELECT or EXISTS) and that the RHS might contains NULLs. -** Furthermore, the IN is in a WHERE clause and that we really want -** to iterate over the RHS of the IN operator in order to quickly locate -** all corresponding LHS elements. All this routine does is initialize -** the register given by rMayHaveNull to NULL. Calling routines will take -** care of changing this register value to non-NULL if the RHS is NULL-free. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** If rMayHaveNull is zero, that means that the subquery is being used -** for membership testing only. There is no need to initialize any -** registers to indicate the presense or absence of NULLs on the RHS. +************************************************************************* ** -** For a SELECT or EXISTS operator, return the register that holds the -** result. For IN operators or if an error occurs, the return value is 0. +** This file contains code use to implement an in-memory rollback journal. +** The in-memory rollback journal is used to journal transactions for +** ":memory:" databases and when the journal_mode=MEMORY pragma is used. */ -#ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3CodeSubselect( - Parse *pParse, /* Parsing context */ - Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ - int rMayHaveNull, /* Register that records whether NULLs exist in RHS */ - int isRowid /* If true, LHS of IN operator is a rowid */ -){ - int testAddr = 0; /* One-time test address */ - int rReg = 0; /* Register storing resulting */ - Vdbe *v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return 0; - sqlite3ExprCachePush(pParse); - /* This code must be run in its entirety every time it is encountered - ** if any of the following is true: - ** - ** * The right-hand side is a correlated subquery - ** * The right-hand side is an expression list containing variables - ** * We are inside a trigger - ** - ** If all of the above are false, then we can run this code just once - ** save the results, and reuse the same result on subsequent invocations. - */ - if( !ExprHasAnyProperty(pExpr, EP_VarSelect) && !pParse->pTriggerTab ){ - int mem = ++pParse->nMem; - sqlite3VdbeAddOp1(v, OP_If, mem); - testAddr = sqlite3VdbeAddOp2(v, OP_Integer, 1, mem); - assert( testAddr>0 || pParse->db->mallocFailed ); - } - -#ifndef SQLITE_OMIT_EXPLAIN - if( pParse->explain==2 ){ - char *zMsg = sqlite3MPrintf( - pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ", - pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId - ); - sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); - } -#endif - - switch( pExpr->op ){ - case TK_IN: { - char affinity; /* Affinity of the LHS of the IN */ - KeyInfo keyInfo; /* Keyinfo for the generated table */ - int addr; /* Address of OP_OpenEphemeral instruction */ - Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ - - if( rMayHaveNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); - } - - affinity = sqlite3ExprAffinity(pLeft); - - /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' - ** expression it is handled the same way. An ephemeral table is - ** filled with single-field index keys representing the results - ** from the SELECT or the . - ** - ** If the 'x' expression is a column value, or the SELECT... - ** statement returns a column value, then the affinity of that - ** column is used to build the index keys. If both 'x' and the - ** SELECT... statement are columns, then numeric affinity is used - ** if either column has NUMERIC or INTEGER affinity. If neither - ** 'x' nor the SELECT... statement are columns, then numeric affinity - ** is used. - */ - pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - memset(&keyInfo, 0, sizeof(keyInfo)); - keyInfo.nField = 1; - - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* Case 1: expr IN (SELECT ...) - ** - ** Generate code to write the results of the select into the temporary - ** table allocated and opened above. - */ - SelectDest dest; - ExprList *pEList; +/* Forward references to internal structures */ +typedef struct MemJournal MemJournal; +typedef struct FilePoint FilePoint; +typedef struct FileChunk FileChunk; - assert( !isRowid ); - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affinity = (u8)affinity; - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pExpr->x.pSelect->iLimit = 0; - if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ - return 0; - } - pEList = pExpr->x.pSelect->pEList; - if( ALWAYS(pEList!=0 && pEList->nExpr>0) ){ - keyInfo.aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, - pEList->a[0].pExpr); - } - }else if( ALWAYS(pExpr->x.pList!=0) ){ - /* Case 2: expr IN (exprlist) - ** - ** For each expression, build an index key from the evaluation and - ** store it in the temporary table. If is a column, then use - ** that columns affinity when building index keys. If is not - ** a column, use numeric affinity. - */ - int i; - ExprList *pList = pExpr->x.pList; - struct ExprList_item *pItem; - int r1, r2, r3; +/* Space to hold the rollback journal is allocated in increments of +** this many bytes. +** +** The size chosen is a little less than a power of two. That way, +** the FileChunk object will have a size that almost exactly fills +** a power-of-two allocation. This mimimizes wasted space in power-of-two +** memory allocators. +*/ +#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) - if( !affinity ){ - affinity = SQLITE_AFF_NONE; - } - keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); +/* Macro to find the minimum of two numeric values. +*/ +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif - /* Loop through each expression in . */ - r1 = sqlite3GetTempReg(pParse); - r2 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Null, 0, r2); - for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ - Expr *pE2 = pItem->pExpr; - int iValToIns; +/* +** The rollback journal is composed of a linked list of these structures. +*/ +struct FileChunk { + FileChunk *pNext; /* Next chunk in the journal */ + u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ +}; - /* If the expression is not constant then we will need to - ** disable the test that was generated above that makes sure - ** this code only executes once. Because for a non-constant - ** expression we need to rerun this code each time. - */ - if( testAddr && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, testAddr-1, 2); - testAddr = 0; - } +/* +** An instance of this object serves as a cursor into the rollback journal. +** The cursor can be either for reading or writing. +*/ +struct FilePoint { + sqlite3_int64 iOffset; /* Offset from the beginning of the file */ + FileChunk *pChunk; /* Specific chunk into which cursor points */ +}; - /* Evaluate the expression and insert it into the temp table */ - if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){ - sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns); - }else{ - r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); - if( isRowid ){ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, - sqlite3VdbeCurrentAddr(v)+2); - sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); - }else{ - sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, r3, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pExpr->iTable, r2); - } - } - } - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempReg(pParse, r2); - } - if( !isRowid ){ - sqlite3VdbeChangeP4(v, addr, (void *)&keyInfo, P4_KEYINFO); - } - break; - } +/* +** This subclass is a subclass of sqlite3_file. Each open memory-journal +** is an instance of this class. +*/ +struct MemJournal { + sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + FileChunk *pFirst; /* Head of in-memory chunk-list */ + FilePoint endpoint; /* Pointer to the end of the file */ + FilePoint readpoint; /* Pointer to the end of the last xRead() */ +}; - case TK_EXISTS: - case TK_SELECT: - default: { - /* If this has to be a scalar SELECT. Generate code to put the - ** value of this select in a memory cell and record the number - ** of the memory cell in iColumn. If this is an EXISTS, write - ** an integer 0 (not exists) or 1 (exists) into a memory cell - ** and record that memory cell in iColumn. - */ - Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECt result */ +/* +** Read data from the in-memory journal file. This is the implementation +** of the sqlite3_vfs.xRead method. +*/ +static int memjrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + MemJournal *p = (MemJournal *)pJfd; + u8 *zOut = zBuf; + int nRead = iAmt; + int iChunkOffset; + FileChunk *pChunk; - testcase( pExpr->op==TK_EXISTS ); - testcase( pExpr->op==TK_SELECT ); - assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + /* SQLite never tries to read past the end of a rollback journal file */ + assert( iOfst+iAmt<=p->endpoint.iOffset ); - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); - pSel = pExpr->x.pSelect; - sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); - if( pExpr->op==TK_SELECT ){ - dest.eDest = SRT_Mem; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iParm); - VdbeComment((v, "Init subquery result")); - }else{ - dest.eDest = SRT_Exists; - sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iParm); - VdbeComment((v, "Init EXISTS result")); - } - sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, - &sqlite3IntTokens[1]); - pSel->iLimit = 0; - if( sqlite3Select(pParse, pSel, &dest) ){ - return 0; - } - rReg = dest.iParm; - ExprSetIrreducible(pExpr); - break; + if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ + sqlite3_int64 iOff = 0; + for(pChunk=p->pFirst; + ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; + pChunk=pChunk->pNext + ){ + iOff += JOURNAL_CHUNKSIZE; } + }else{ + pChunk = p->readpoint.pChunk; } - if( testAddr ){ - sqlite3VdbeJumpHere(v, testAddr-1); - } - sqlite3ExprCachePop(pParse, 1); + iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); + do { + int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; + int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); + memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + zOut += nCopy; + nRead -= iSpace; + iChunkOffset = 0; + } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); + p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.pChunk = pChunk; - return rReg; + return SQLITE_OK; } -#endif /* SQLITE_OMIT_SUBQUERY */ -#ifndef SQLITE_OMIT_SUBQUERY /* -** Generate code for an IN expression. -** -** x IN (SELECT ...) -** x IN (value, value, ...) -** -** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) -** is an array of zero or more values. The expression is true if the LHS is -** contained within the RHS. The value of the expression is unknown (NULL) -** if the LHS is NULL or if the LHS is not contained within the RHS and the -** RHS contains one or more NULL values. -** -** This routine generates code will jump to destIfFalse if the LHS is not -** contained within the RHS. If due to NULLs we cannot determine if the LHS -** is contained in the RHS then jump to destIfNull. If the LHS is contained -** within the RHS then fall through. +** Write data to the file. */ -static void sqlite3ExprCodeIN( - Parse *pParse, /* Parsing and code generating context */ - Expr *pExpr, /* The IN expression */ - int destIfFalse, /* Jump here if LHS is not contained in the RHS */ - int destIfNull /* Jump here if the results are unknown due to NULLs */ +static int memjrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ ){ - int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ - char affinity; /* Comparison affinity to use */ - int eType; /* Type of the RHS */ - int r1; /* Temporary use register */ - Vdbe *v; /* Statement under construction */ + MemJournal *p = (MemJournal *)pJfd; + int nWrite = iAmt; + u8 *zWrite = (u8 *)zBuf; - /* Compute the RHS. After this step, the table with cursor - ** pExpr->iTable will contains the values that make up the RHS. + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required by sqlite. */ - v = pParse->pVdbe; - assert( v!=0 ); /* OOM detected prior to this routine */ - VdbeNoopComment((v, "begin IN expr")); - eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull); + assert( iOfst==p->endpoint.iOffset ); + UNUSED_PARAMETER(iOfst); - /* Figure out the affinity to use to create a key from the results - ** of the expression. affinityStr stores a static string suitable for - ** P4 of OP_MakeRecord. - */ - affinity = comparisonAffinity(pExpr); + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); + int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); - /* Code the LHS, the from " IN (...)". - */ - sqlite3ExprCachePush(pParse); - r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCode(pParse, pExpr->pLeft, r1); + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. - */ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - sqlite3VdbeJumpHere(v, addr1); + memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; } - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree - */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - if( rRhsHasNull==0 || destIfFalse==destIfNull ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. - */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - - }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. - */ - int j1, j2, j3; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the presence of NULLs in the RHS does not matter, so jump - ** over all of the code that follows. - */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - - /* Here we begin generating code that runs if the LHS is not - ** contained within the RHS. Generate additional code that - ** tests the RHS for NULLs. If the RHS contains a NULL then - ** jump to destIfNull. If there are no NULLs in the RHS then - ** jump to destIfFalse. - */ - j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull); - j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); - sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull); - sqlite3VdbeJumpHere(v, j3); - sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1); - sqlite3VdbeJumpHere(v, j2); - - /* Jump to the appropriate target depending on whether or not - ** the RHS contains a NULL - */ - sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - - /* The OP_Found at the top of this branch jumps here when true, - ** causing the overall IN expression evaluation to fall through. - */ - sqlite3VdbeJumpHere(v, j1); - } - } - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ExprCachePop(pParse, 1); - VdbeComment((v, "end IN expr")); + return SQLITE_OK; } -#endif /* SQLITE_OMIT_SUBQUERY */ /* -** Duplicate an 8-byte value +** Truncate the file. */ -static char *dup8bytes(Vdbe *v, const char *in){ - char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); - if( out ){ - memcpy(out, in, 8); +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + MemJournal *p = (MemJournal *)pJfd; + FileChunk *pChunk; + assert(size==0); + UNUSED_PARAMETER(size); + pChunk = p->pFirst; + while( pChunk ){ + FileChunk *pTmp = pChunk; + pChunk = pChunk->pNext; + sqlite3_free(pTmp); } - return out; + sqlite3MemJournalOpen(pJfd); + return SQLITE_OK; } -#ifndef SQLITE_OMIT_FLOATING_POINT /* -** Generate an instruction that will put the floating point -** value described by z[0..n-1] into register iMem. -** -** The z[] string will probably not be zero-terminated. But the -** z[n] character is guaranteed to be something that does not look -** like the continuation of the number. +** Close the file. */ -static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ - if( ALWAYS(z!=0) ){ - double value; - char *zV; - sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); - assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ - if( negateFlag ) value = -value; - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); - } +static int memjrnlClose(sqlite3_file *pJfd){ + memjrnlTruncate(pJfd, 0); + return SQLITE_OK; } -#endif /* -** Generate an instruction that will put the integer describe by -** text z[0..n-1] into register iMem. +** Sync the file. ** -** Expr.u.zToken is always UTF8 and zero-terminated. +** Syncing an in-memory journal is a no-op. And, in fact, this routine +** is never called in a working implementation. This implementation +** exists purely as a contingency, in case some malfunction in some other +** part of SQLite causes Sync to be called by mistake. */ -static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ - Vdbe *v = pParse->pVdbe; - if( pExpr->flags & EP_IntValue ){ - int i = pExpr->u.iValue; - if( negFlag ) i = -i; - sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); - }else{ - int c; - i64 value; - const char *z = pExpr->u.zToken; - assert( z!=0 ); - c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); - if( c==0 || (c==2 && negFlag) ){ - char *zV; - if( negFlag ){ value = -value; } - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); - }else{ -#ifdef SQLITE_OMIT_FLOATING_POINT - sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); -#else - codeReal(v, z, negFlag, iMem); -#endif - } - } +static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; } /* -** Clear a cache entry. +** Query the size of the file in bytes. */ -static void cacheEntryClear(Parse *pParse, struct yColCache *p){ - if( p->tempReg ){ - if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = p->iReg; - } - p->tempReg = 0; - } +static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + MemJournal *p = (MemJournal *)pJfd; + *pSize = (sqlite_int64) p->endpoint.iOffset; + return SQLITE_OK; } - /* -** Record in the column cache that a particular column from a -** particular table is stored in a particular register. +** Table of methods for MemJournal sqlite3_file object. */ -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ - int i; - int minLru; - int idxLru; - struct yColCache *p; - - assert( iReg>0 ); /* Register numbers are always positive */ - assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ - - /* The SQLITE_ColumnCache flag disables the column cache. This is used - ** for testing only - to verify that SQLite always gets the same answer - ** with and without the column cache. - */ - if( pParse->db->flags & SQLITE_ColumnCache ) return; - - /* First replace any existing entry. - ** - ** Actually, the way the column cache is currently used, we are guaranteed - ** that the object will never already be in cache. Verify this guarantee. - */ -#ifndef NDEBUG - for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif - assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); - } -#endif - - /* Find an empty slot and replace it */ - for(i=0, p=pParse->aColCache; iiReg==0 ){ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; - } - } - - /* Replace the last recently used */ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - if( ALWAYS(idxLru>=0) ){ - p = &pParse->aColCache[idxLru]; - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; - } -} +static const struct sqlite3_io_methods MemJournalMethods = { + 1, /* iVersion */ + memjrnlClose, /* xClose */ + memjrnlRead, /* xRead */ + memjrnlWrite, /* xWrite */ + memjrnlTruncate, /* xTruncate */ + memjrnlSync, /* xSync */ + memjrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0 /* xShmUnlock */ +}; -/* -** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. -** Purge the range of registers from the column cache. +/* +** Open a journal file. */ -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i; - int iLast = iReg + nReg - 1; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iReg && r<=iLast ){ - cacheEntryClear(pParse, p); - p->iReg = 0; - } - } +SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + MemJournal *p = (MemJournal *)pJfd; + assert( EIGHT_BYTE_ALIGNMENT(p) ); + memset(p, 0, sqlite3MemJournalSize()); + p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; } /* -** Remember the current column cache context. Any new entries added -** added to the column cache after this call are removed when the -** corresponding pop occurs. +** Return true if the file-handle passed as an argument is +** an in-memory journal */ -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ - pParse->iCacheLevel++; +SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ + return pJfd->pMethods==&MemJournalMethods; } -/* -** Remove from the column cache any entries that were added since the -** the previous N Push operations. In other words, restore the cache -** to the state it was in N Pushes ago. +/* +** Return the number of bytes required to store a MemJournal file descriptor. */ -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse, int N){ - int i; - struct yColCache *p; - assert( N>0 ); - assert( pParse->iCacheLevel>=N ); - pParse->iCacheLevel -= N; - for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, p); - p->iReg = 0; - } - } +SQLITE_PRIVATE int sqlite3MemJournalSize(void){ + return sizeof(MemJournal); } +/************** End of memjournal.c ******************************************/ +/************** Begin file walker.c ******************************************/ /* -** When a cached column is reused, make sure that its register is -** no longer available as a temp register. ticket #3879: that same -** register might be in the cache in multiple places, so be sure to -** get them all. +** 2008 August 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used for walking the parser tree for +** an SQL statement. */ -static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg==iReg ){ - p->tempReg = 0; - } - } -} -/* -** Generate code to extract the value of the iCol-th column of a table. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( - Vdbe *v, /* The VDBE under construction */ - Table *pTab, /* The table containing the value */ - int iTabCur, /* The cursor for this table */ - int iCol, /* Index of the column to extract */ - int regOut /* Extract the valud into this register */ -){ - if( iCol<0 || iCol==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); - }else{ - int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; - sqlite3VdbeAddOp3(v, op, iTabCur, iCol, regOut); - } - if( iCol>=0 ){ - sqlite3ColumnDefault(v, pTab, iCol, regOut); - } -} /* -** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. An effort -** is made to store the column value in register iReg, but this is -** not guaranteed. The location of the column value is returned. +** Walk an expression tree. Invoke the callback once for each node +** of the expression, while decending. (In other words, the callback +** is invoked before visiting children.) ** -** There must be an open cursor to pTab in iTable when this routine -** is called. If iColumn<0 then code is generated that extracts the rowid. +** The return value from the callback should be one of the WRC_* +** constants to specify how to proceed with the walk. +** +** WRC_Continue Continue descending down the tree. +** +** WRC_Prune Do not descend into child nodes. But allow +** the walk to continue with sibling nodes. +** +** WRC_Abort Do no more callbacks. Unwind the stack and +** return the top-level walk call. +** +** The return value from this routine is WRC_Abort to abandon the tree walk +** and WRC_Continue to continue. */ -SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( - Parse *pParse, /* Parsing and code generating context */ - Table *pTab, /* Description of the table we are reading from */ - int iColumn, /* Index of the table column */ - int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ -){ - Vdbe *v = pParse->pVdbe; - int i; - struct yColCache *p; - - for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ - p->lru = pParse->iCacheCnt++; - sqlite3ExprCachePinRegister(pParse, p->iReg); - return p->iReg; +SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + int rc; + if( pExpr==0 ) return WRC_Continue; + testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); + testcase( ExprHasProperty(pExpr, EP_Reduced) ); + rc = pWalker->xExprCallback(pWalker, pExpr); + if( rc==WRC_Continue + && !ExprHasAnyProperty(pExpr,EP_TokenOnly) ){ + if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else{ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; } - } - assert( v!=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); - return iReg; + } + return rc & WRC_Abort; } /* -** Clear all column cache entries. +** Call sqlite3WalkExpr() for every expression in list p or until +** an abort request is seen. */ -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ +SQLITE_PRIVATE int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ int i; - struct yColCache *p; - - for(i=0, p=pParse->aColCache; iiReg ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + struct ExprList_item *pItem; + if( p ){ + for(i=p->nExpr, pItem=p->a; i>0; i--, pItem++){ + if( sqlite3WalkExpr(pWalker, pItem->pExpr) ) return WRC_Abort; } } + return WRC_Continue; } /* -** Record the fact that an affinity change has occurred on iCount -** registers starting with iStart. +** Walk all expressions associated with SELECT statement p. Do +** not invoke the SELECT callback on p, but do (of course) invoke +** any expr callbacks and SELECT callbacks that come from subqueries. +** Return WRC_Abort or WRC_Continue. */ -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ - sqlite3ExprCacheRemove(pParse, iStart, iCount); +SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ + if( sqlite3WalkExprList(pWalker, p->pEList) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pWhere) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, p->pGroupBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pOffset) ) return WRC_Abort; + return WRC_Continue; } /* -** Generate code to move content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. +** Walk the parse trees associated with all subqueries in the +** FROM clause of SELECT statement p. Do not invoke the select +** callback on p, but do invoke it on each FROM clause subquery +** and on any subqueries further down in the tree. Return +** WRC_Abort or WRC_Continue; */ -SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ +SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ + SrcList *pSrc; int i; - struct yColCache *p; - if( NEVER(iFrom==iTo) ) return; - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - for(i=0, p=pParse->aColCache; iiReg; - if( x>=iFrom && xiReg += iTo-iFrom; + struct SrcList_item *pItem; + + pSrc = p->pSrc; + if( ALWAYS(pSrc) ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + return WRC_Abort; + } } } -} + return WRC_Continue; +} /* -** Generate code to copy content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. +** Call sqlite3WalkExpr() for every expression in Select statement p. +** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and +** on the compound select chain, p->pPrior. +** +** Return WRC_Continue under normal conditions. Return WRC_Abort if +** there is an abort request. +** +** If the Walker does not have an xSelectCallback() then this routine +** is a no-op returning WRC_Continue. */ -SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - if( NEVER(iFrom==iTo) ) return; - for(i=0; ipVdbe, OP_Copy, iFrom+i, iTo+i); +SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ + int rc; + if( p==0 || pWalker->xSelectCallback==0 ) return WRC_Continue; + rc = WRC_Continue; + while( p ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + if( sqlite3WalkSelectExpr(pWalker, p) ) return WRC_Abort; + if( sqlite3WalkSelectFrom(pWalker, p) ) return WRC_Abort; + p = p->pPrior; } + return rc & WRC_Abort; } -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +/************** End of walker.c **********************************************/ +/************** Begin file resolve.c *****************************************/ /* -** Return true if any register in the range iFrom..iTo (inclusive) -** is used as part of the column cache. +** 2008 August 18 ** -** This routine is used within assert() and testcase() macros only -** and does not appear in a normal build. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains routines used for walking the parser tree and +** resolve all identifiers by associating them with a particular +** table and column. */ -static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ + +/* +** Turn the pExpr expression into an alias for the iCol-th column of the +** result set in pEList. +** +** If the result set column is a simple column reference, then this routine +** makes an exact copy. But for any other kind of expression, this +** routine make a copy of the result set column as the argument to the +** TK_AS operator. The TK_AS operator causes the expression to be +** evaluated just once and then reused for each alias. +** +** The reason for suppressing the TK_AS term when the expression is a simple +** column reference is so that the column reference will be recognized as +** usable by indices within the WHERE clause processing logic. +** +** Hack: The TK_AS operator is inhibited if zType[0]=='G'. This means +** that in a GROUP BY clause, the expression is evaluated twice. Hence: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x +** +** Is equivalent to: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 +** +** The result of random()%5 in the GROUP BY clause is probably different +** from the result in the result-set. We might fix this someday. Or +** then again, we might not... +*/ +static void resolveAlias( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* A result set */ + int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ + Expr *pExpr, /* Transform this into an alias to the result set */ + const char *zType /* "GROUP" or "ORDER" or "" */ +){ + Expr *pOrig; /* The iCol-th column of the result set */ + Expr *pDup; /* Copy of pOrig */ + sqlite3 *db; /* The database connection */ + + assert( iCol>=0 && iColnExpr ); + pOrig = pEList->a[iCol].pExpr; + assert( pOrig!=0 ); + assert( pOrig->flags & EP_Resolved ); + db = pParse->db; + if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ + pDup = sqlite3ExprDup(db, pOrig, 0); + pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); + if( pDup==0 ) return; + if( pEList->a[iCol].iAlias==0 ){ + pEList->a[iCol].iAlias = (u16)(++pParse->nAlias); + } + pDup->iTable = pEList->a[iCol].iAlias; + }else if( ExprHasProperty(pOrig, EP_IntValue) || pOrig->u.zToken==0 ){ + pDup = sqlite3ExprDup(db, pOrig, 0); + if( pDup==0 ) return; + }else{ + char *zToken = pOrig->u.zToken; + assert( zToken!=0 ); + pOrig->u.zToken = 0; + pDup = sqlite3ExprDup(db, pOrig, 0); + pOrig->u.zToken = zToken; + if( pDup==0 ) return; + assert( (pDup->flags & (EP_Reduced|EP_TokenOnly))==0 ); + pDup->flags2 |= EP2_MallocedToken; + pDup->u.zToken = sqlite3DbStrDup(db, zToken); } - return 0; + if( pExpr->flags & EP_ExpCollate ){ + pDup->pColl = pExpr->pColl; + pDup->flags |= EP_ExpCollate; + } + + /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This + ** prevents ExprDelete() from deleting the Expr structure itself, + ** allowing it to be repopulated by the memcpy() on the following line. + */ + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(db, pExpr); + memcpy(pExpr, pDup, sizeof(*pExpr)); + sqlite3DbFree(db, pDup); } -#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ /* -** Generate code into the current Vdbe to evaluate the given -** expression. Attempt to store the results in register "target". -** Return the register where results are stored. +** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up +** that name in the set of source tables in pSrcList and make the pExpr +** expression node refer back to that source column. The following changes +** are made to pExpr: ** -** With this routine, there is no guarantee that results will -** be stored in target. The result might be stored in some other -** register if it is convenient to do so. The calling function -** must check the return code and move the results to the desired -** register. +** pExpr->iDb Set the index in db->aDb[] of the database X +** (even if X is implied). +** pExpr->iTable Set to the cursor number for the table obtained +** from pSrcList. +** pExpr->pTab Points to the Table structure of X.Y (even if +** X and/or Y are implied.) +** pExpr->iColumn Set to the column number within the table. +** pExpr->op Set to TK_COLUMN. +** pExpr->pLeft Any expression this points to is deleted +** pExpr->pRight Any expression this points to is deleted. +** +** The zDb variable is the name of the database (the "X"). This value may be +** NULL meaning that name is of the form Y.Z or Z. Any available database +** can be used. The zTable variable is the name of the table (the "Y"). This +** value can be NULL if zDb is also NULL. If zTable is NULL it +** means that the form of the name is Z and that columns from any table +** can be used. +** +** If the name cannot be resolved unambiguously, leave an error message +** in pParse and return WRC_Abort. Return WRC_Prune on success. */ -SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ - Vdbe *v = pParse->pVdbe; /* The VM under construction */ - int op; /* The opcode being coded */ - int inReg = target; /* Results stored in register inReg */ - int regFree1 = 0; /* If non-zero free this temporary register */ - int regFree2 = 0; /* If non-zero free this temporary register */ - int r1, r2, r3, r4; /* Various register numbers */ - sqlite3 *db = pParse->db; /* The database connection */ +static int lookupName( + Parse *pParse, /* The parsing context */ + const char *zDb, /* Name of the database containing table, or NULL */ + const char *zTab, /* Name of table containing column, or NULL */ + const char *zCol, /* Name of the column. */ + NameContext *pNC, /* The name context used to resolve the name */ + Expr *pExpr /* Make this EXPR node point to the selected column */ +){ + int i, j; /* Loop counters */ + int cnt = 0; /* Number of matching column names */ + int cntTab = 0; /* Number of matching table names */ + sqlite3 *db = pParse->db; /* The database connection */ + struct SrcList_item *pItem; /* Use for looping over pSrcList items */ + struct SrcList_item *pMatch = 0; /* The matching pSrcList item */ + NameContext *pTopNC = pNC; /* First namecontext in the list */ + Schema *pSchema = 0; /* Schema of the expression */ + int isTrigger = 0; - assert( target>0 && target<=pParse->nMem ); - if( v==0 ){ - assert( pParse->db->mallocFailed ); - return 0; - } + assert( pNC ); /* the name context cannot be NULL. */ + assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ + assert( ~ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); - if( pExpr==0 ){ - op = TK_NULL; - }else{ - op = pExpr->op; - } - switch( op ){ - case TK_AGG_COLUMN: { - AggInfo *pAggInfo = pExpr->pAggInfo; - struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; - if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - inReg = pCol->iMem; - break; - }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, - pCol->iSorterColumn, target); - break; + /* Initialize the node to no-match */ + pExpr->iTable = -1; + pExpr->pTab = 0; + ExprSetIrreducible(pExpr); + + /* Start at the inner-most context and move outward until a match is found */ + while( pNC && cnt==0 ){ + ExprList *pEList; + SrcList *pSrcList = pNC->pSrcList; + + if( pSrcList ){ + for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ + Table *pTab; + int iDb; + Column *pCol; + + pTab = pItem->pTab; + assert( pTab!=0 && pTab->zName!=0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( pTab->nCol>0 ); + if( zTab ){ + if( pItem->zAlias ){ + char *zTabName = pItem->zAlias; + if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; + }else{ + char *zTabName = pTab->zName; + if( NEVER(zTabName==0) || sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } + if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ + continue; + } + } + } + if( 0==(cntTab++) ){ + pExpr->iTable = pItem->iCursor; + pExpr->pTab = pTab; + pSchema = pTab->pSchema; + pMatch = pItem; + } + for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + IdList *pUsing; + cnt++; + pExpr->iTable = pItem->iCursor; + pExpr->pTab = pTab; + pMatch = pItem; + pSchema = pTab->pSchema; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + if( inSrc-1 ){ + if( pItem[1].jointype & JT_NATURAL ){ + /* If this match occurred in the left table of a natural join, + ** then skip the right table to avoid a duplicate match */ + pItem++; + i++; + }else if( (pUsing = pItem[1].pUsing)!=0 ){ + /* If this match occurs on a column that is in the USING clause + ** of a join, skip the search of the right table of the join + ** to avoid a duplicate match there. */ + int k; + for(k=0; knId; k++){ + if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ){ + pItem++; + i++; + break; + } + } + } + } + break; + } + } } - /* Otherwise, fall thru into the TK_COLUMN case */ } - case TK_COLUMN: { - if( pExpr->iTable<0 ){ - /* This only happens when coding check constraints */ - assert( pParse->ckBase>0 ); - inReg = pExpr->iColumn + pParse->ckBase; - }else{ - inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + +#ifndef SQLITE_OMIT_TRIGGER + /* If we have not already resolved the name, then maybe + ** it is a new.* or old.* trigger argument reference + */ + if( zDb==0 && zTab!=0 && cnt==0 && pParse->pTriggerTab!=0 ){ + int op = pParse->eTriggerOp; + Table *pTab = 0; + assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); + if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){ + pExpr->iTable = 1; + pTab = pParse->pTriggerTab; + }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){ + pExpr->iTable = 0; + pTab = pParse->pTriggerTab; + } + + if( pTab ){ + int iCol; + pSchema = pTab->pSchema; + cntTab++; + for(iCol=0; iColnCol; iCol++){ + Column *pCol = &pTab->aCol[iCol]; + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( iCol==pTab->iPKey ){ + iCol = -1; + } + break; + } + } + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) ){ + iCol = -1; /* IMP: R-44911-55124 */ + } + if( iColnCol ){ + cnt++; + if( iCol<0 ){ + pExpr->affinity = SQLITE_AFF_INTEGER; + }else if( pExpr->iTable==0 ){ + testcase( iCol==31 ); + testcase( iCol==32 ); + pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<iColumn = (i16)iCol; + pExpr->pTab = pTab; + isTrigger = 1; + } } - break; } - case TK_INTEGER: { - codeInteger(pParse, pExpr, 0, target); - break; +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ + + /* + ** Perhaps the name is a reference to the ROWID + */ + if( cnt==0 && cntTab==1 && sqlite3IsRowid(zCol) ){ + cnt = 1; + pExpr->iColumn = -1; /* IMP: R-44911-55124 */ + pExpr->affinity = SQLITE_AFF_INTEGER; } -#ifndef SQLITE_OMIT_FLOATING_POINT - case TK_FLOAT: { - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - codeReal(v, pExpr->u.zToken, 0, target); - break; + + /* + ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z + ** might refer to an result-set alias. This happens, for example, when + ** we are resolving names in the WHERE clause of the following command: + ** + ** SELECT a+b AS x FROM table WHERE x<10; + ** + ** In cases like this, replace pExpr with a copy of the expression that + ** forms the result set entry ("a+b" in the example) and return immediately. + ** Note that the expression in the result set should have already been + ** resolved by the time the WHERE clause is resolved. + */ + if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + for(j=0; jnExpr; j++){ + char *zAs = pEList->a[j].zName; + if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + Expr *pOrig; + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->x.pList==0 ); + assert( pExpr->x.pSelect==0 ); + pOrig = pEList->a[j].pExpr; + if( !pNC->allowAgg && ExprHasProperty(pOrig, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); + return WRC_Abort; + } + resolveAlias(pParse, pEList, j, pExpr, ""); + cnt = 1; + pMatch = 0; + assert( zTab==0 && zDb==0 ); + goto lookupname_end; + } + } + } + + /* Advance to the next name context. The loop will exit when either + ** we have a match (cnt>0) or when we run out of name contexts. + */ + if( cnt==0 ){ + pNC = pNC->pNext; + } + } + + /* + ** If X and Y are NULL (in other words if only the column name Z is + ** supplied) and the value of Z is enclosed in double-quotes, then + ** Z is a string literal if it doesn't match any column names. In that + ** case, we need to return right away and not make any changes to + ** pExpr. + ** + ** Because no reference was made to outer contexts, the pNC->nRef + ** fields are not changed in any context. + */ + if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ + pExpr->op = TK_STRING; + pExpr->pTab = 0; + return WRC_Prune; + } + + /* + ** cnt==0 means there was not match. cnt>1 means there were two or + ** more matches. Either way, we have an error. + */ + if( cnt!=1 ){ + const char *zErr; + zErr = cnt==0 ? "no such column" : "ambiguous column name"; + if( zDb ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); + }else if( zTab ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); + } + pParse->checkSchema = 1; + pTopNC->nErr++; + } + + /* If a column from a table in pSrcList is referenced, then record + ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes + ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the + ** column number is greater than the number of bits in the bitmask + ** then set the high-order bit of the bitmask. + */ + if( pExpr->iColumn>=0 && pMatch!=0 ){ + int n = pExpr->iColumn; + testcase( n==BMS-1 ); + if( n>=BMS ){ + n = BMS-1; + } + assert( pMatch->iCursor==pExpr->iTable ); + pMatch->colUsed |= ((Bitmask)1)<pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN); +lookupname_end: + if( cnt==1 ){ + assert( pNC!=0 ); + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + /* Increment the nRef value on all name contexts from TopNC up to + ** the point where the name matched. */ + for(;;){ + assert( pTopNC!=0 ); + pTopNC->nRef++; + if( pTopNC==pNC ) break; + pTopNC = pTopNC->pNext; + } + return WRC_Prune; + } else { + return WRC_Abort; + } +} + +/* +** Allocate and return a pointer to an expression to load the column iCol +** from datasource iSrc in SrcList pSrc. +*/ +SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ + Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); + if( p ){ + struct SrcList_item *pItem = &pSrc->a[iSrc]; + p->pTab = pItem->pTab; + p->iTable = pItem->iCursor; + if( p->pTab->iPKey==iCol ){ + p->iColumn = -1; + }else{ + p->iColumn = (ynVar)iCol; + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + ExprSetProperty(p, EP_Resolved); + } + return p; +} + +/* +** This routine is callback for sqlite3WalkExpr(). +** +** Resolve symbolic names into TK_COLUMN operators for the current +** node in the expression tree. Return 0 to continue the search down +** the tree or 2 to abort the tree walk. +** +** This routine also does error checking and name resolution for +** function names. The operator for aggregate functions is changed +** to TK_AGG_FUNCTION. +*/ +static int resolveExprStep(Walker *pWalker, Expr *pExpr){ + NameContext *pNC; + Parse *pParse; + + pNC = pWalker->u.pNC; + assert( pNC!=0 ); + pParse = pNC->pParse; + assert( pParse==pWalker->pParse ); + + if( ExprHasAnyProperty(pExpr, EP_Resolved) ) return WRC_Prune; + ExprSetProperty(pExpr, EP_Resolved); +#ifndef NDEBUG + if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ + SrcList *pSrcList = pNC->pSrcList; + int i; + for(i=0; ipSrcList->nSrc; i++){ + assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); } + } #endif - case TK_STRING: { - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + switch( pExpr->op ){ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) + /* The special operator TK_ROW means use the rowid for the first + ** column in the FROM clause. This is used by the LIMIT and ORDER BY + ** clause processing on UPDATE and DELETE statements. + */ + case TK_ROW: { + SrcList *pSrcList = pNC->pSrcList; + struct SrcList_item *pItem; + assert( pSrcList && pSrcList->nSrc==1 ); + pItem = pSrcList->a; + pExpr->op = TK_COLUMN; + pExpr->pTab = pItem->pTab; + pExpr->iTable = pItem->iCursor; + pExpr->iColumn = -1; + pExpr->affinity = SQLITE_AFF_INTEGER; break; } - case TK_NULL: { - sqlite3VdbeAddOp2(v, OP_Null, 0, target); - break; +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ + + /* A lone identifier is the name of a column. + */ + case TK_ID: { + return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); } -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: { - int n; - const char *z; - char *zBlob; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); - assert( pExpr->u.zToken[1]=='\'' ); - z = &pExpr->u.zToken[2]; - n = sqlite3Strlen30(z) - 1; - assert( z[n]=='\'' ); - zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); - sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); - break; + + /* A table name and column name: ID.ID + ** Or a database, table and column: ID.ID.ID + */ + case TK_DOT: { + const char *zColumn; + const char *zTable; + const char *zDb; + Expr *pRight; + + /* if( pSrcList==0 ) break; */ + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + zTable = pExpr->pLeft->u.zToken; + zColumn = pRight->u.zToken; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pExpr->pLeft->u.zToken; + zTable = pRight->pLeft->u.zToken; + zColumn = pRight->pRight->u.zToken; + } + return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); } + + /* Resolve function names + */ + case TK_CONST_FUNC: + case TK_FUNCTION: { + ExprList *pList = pExpr->x.pList; /* The argument list */ + int n = pList ? pList->nExpr : 0; /* Number of arguments */ + int no_such_func = 0; /* True if no such function exists */ + int wrong_num_args = 0; /* True if wrong number of arguments */ + int is_agg = 0; /* True if is an aggregate function */ + int auth; /* Authorization to use the function */ + int nId; /* Number of characters in function name */ + const char *zId; /* The function name. */ + FuncDef *pDef; /* Information about the function */ + u8 enc = ENC(pParse->db); /* The database encoding */ + + testcase( pExpr->op==TK_CONST_FUNC ); + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + zId = pExpr->u.zToken; + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); + if( pDef==0 ){ + pDef = sqlite3FindFunction(pParse->db, zId, nId, -1, enc, 0); + if( pDef==0 ){ + no_such_func = 1; + }else{ + wrong_num_args = 1; + } + }else{ + is_agg = pDef->xFunc==0; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pDef ){ + auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; + } + } #endif - case TK_VARIABLE: { - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - assert( pExpr->u.zToken!=0 ); - assert( pExpr->u.zToken[0]!=0 ); - sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - sqlite3VdbeChangeP4(v, -1, pExpr->u.zToken, 0); + if( is_agg && !pNC->allowAgg ){ + sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); + pNC->nErr++; + is_agg = 0; + }else if( no_such_func ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nErr++; } - break; + if( is_agg ){ + pExpr->op = TK_AGG_FUNCTION; + pNC->hasAgg = 1; + } + if( is_agg ) pNC->allowAgg = 0; + sqlite3WalkExprList(pWalker, pList); + if( is_agg ) pNC->allowAgg = 1; + /* FIX ME: Compute pExpr->affinity based on the expected return + ** type of the function + */ + return WRC_Prune; } - case TK_REGISTER: { - inReg = pExpr->iTable; +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: + case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); +#endif + case TK_IN: { + testcase( pExpr->op==TK_IN ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + int nRef = pNC->nRef; +#ifndef SQLITE_OMIT_CHECK + if( pNC->isCheck ){ + sqlite3ErrorMsg(pParse,"subqueries prohibited in CHECK constraints"); + } +#endif + sqlite3WalkSelect(pWalker, pExpr->x.pSelect); + assert( pNC->nRef>=nRef ); + if( nRef!=pNC->nRef ){ + ExprSetProperty(pExpr, EP_VarSelect); + } + } break; } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); +#ifndef SQLITE_OMIT_CHECK + case TK_VARIABLE: { + if( pNC->isCheck ){ + sqlite3ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); + } break; } -#ifndef SQLITE_OMIT_CAST - case TK_CAST: { - /* Expressions of the form: CAST(pLeft AS token) */ - int aff, to_op; - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - aff = sqlite3AffinityType(pExpr->u.zToken); - to_op = aff - SQLITE_AFF_TEXT + OP_ToText; - assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); - assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); - assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); - assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); - assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); - testcase( to_op==OP_ToText ); - testcase( to_op==OP_ToBlob ); - testcase( to_op==OP_ToNumeric ); - testcase( to_op==OP_ToInt ); - testcase( to_op==OP_ToReal ); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } - sqlite3VdbeAddOp1(v, to_op, inReg); - testcase( usedAsColumnCache(pParse, inReg, inReg) ); - sqlite3ExprCacheAffinityChange(pParse, inReg, 1); - break; - } -#endif /* SQLITE_OMIT_CAST */ - case TK_LT: - case TK_LE: - case TK_GT: - case TK_GE: - case TK_NE: - case TK_EQ: { - assert( TK_LT==OP_Lt ); - assert( TK_LE==OP_Le ); - assert( TK_GT==OP_Gt ); - assert( TK_GE==OP_Ge ); - assert( TK_EQ==OP_Eq ); - assert( TK_NE==OP_Ne ); - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_AND: - case TK_OR: - case TK_PLUS: - case TK_STAR: - case TK_MINUS: - case TK_REM: - case TK_BITAND: - case TK_BITOR: - case TK_SLASH: - case TK_LSHIFT: - case TK_RSHIFT: - case TK_CONCAT: { - assert( TK_AND==OP_And ); - assert( TK_OR==OP_Or ); - assert( TK_PLUS==OP_Add ); - assert( TK_MINUS==OP_Subtract ); - assert( TK_REM==OP_Remainder ); - assert( TK_BITAND==OP_BitAnd ); - assert( TK_BITOR==OP_BitOr ); - assert( TK_SLASH==OP_Divide ); - assert( TK_LSHIFT==OP_ShiftLeft ); - assert( TK_RSHIFT==OP_ShiftRight ); - assert( TK_CONCAT==OP_Concat ); - testcase( op==TK_AND ); - testcase( op==TK_OR ); - testcase( op==TK_PLUS ); - testcase( op==TK_MINUS ); - testcase( op==TK_REM ); - testcase( op==TK_BITAND ); - testcase( op==TK_BITOR ); - testcase( op==TK_SLASH ); - testcase( op==TK_LSHIFT ); - testcase( op==TK_RSHIFT ); - testcase( op==TK_CONCAT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - sqlite3VdbeAddOp3(v, op, r2, r1, target); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_UMINUS: { - Expr *pLeft = pExpr->pLeft; - assert( pLeft ); - if( pLeft->op==TK_INTEGER ){ - codeInteger(pParse, pLeft, 1, target); -#ifndef SQLITE_OMIT_FLOATING_POINT - }else if( pLeft->op==TK_FLOAT ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - codeReal(v, pLeft->u.zToken, 1, target); #endif - }else{ - regFree1 = r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Integer, 0, r1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); - sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); - testcase( regFree2==0 ); + } + return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; +} + +/* +** pEList is a list of expressions which are really the result set of the +** a SELECT statement. pE is a term in an ORDER BY or GROUP BY clause. +** This routine checks to see if pE is a simple identifier which corresponds +** to the AS-name of one of the terms of the expression list. If it is, +** this routine return an integer between 1 and N where N is the number of +** elements in pEList, corresponding to the matching entry. If there is +** no match, or if pE is not a simple identifier, then this routine +** return 0. +** +** pEList has been resolved. pE has not. +*/ +static int resolveAsName( + Parse *pParse, /* Parsing context for error messages */ + ExprList *pEList, /* List of expressions to scan */ + Expr *pE /* Expression we are trying to match */ +){ + int i; /* Loop counter */ + + UNUSED_PARAMETER(pParse); + + if( pE->op==TK_ID ){ + char *zCol = pE->u.zToken; + for(i=0; inExpr; i++){ + char *zAs = pEList->a[i].zName; + if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + return i+1; } - inReg = target; - break; - } - case TK_BITNOT: - case TK_NOT: { - assert( TK_BITNOT==OP_BitNot ); - assert( TK_NOT==OP_Not ); - testcase( op==TK_BITNOT ); - testcase( op==TK_NOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - testcase( regFree1==0 ); - inReg = target; - sqlite3VdbeAddOp2(v, op, r1, inReg); - break; - } - case TK_ISNULL: - case TK_NOTNULL: { - int addr; - assert( TK_ISNULL==OP_IsNull ); - assert( TK_NOTNULL==OP_NotNull ); - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); - sqlite3VdbeAddOp2(v, OP_Integer, 1, target); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - testcase( regFree1==0 ); - addr = sqlite3VdbeAddOp1(v, op, r1); - sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); - sqlite3VdbeJumpHere(v, addr); - break; } - case TK_AGG_FUNCTION: { - AggInfo *pInfo = pExpr->pAggInfo; - if( pInfo==0 ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); - }else{ - inReg = pInfo->aFunc[pExpr->iAgg].iMem; - } - break; + } + return 0; +} + +/* +** pE is a pointer to an expression which is a single term in the +** ORDER BY of a compound SELECT. The expression has not been +** name resolved. +** +** At the point this routine is called, we already know that the +** ORDER BY term is not an integer index into the result set. That +** case is handled by the calling routine. +** +** Attempt to match pE against result set columns in the left-most +** SELECT statement. Return the index i of the matching column, +** as an indication to the caller that it should sort by the i-th column. +** The left-most column is 1. In other words, the value returned is the +** same integer value that would be used in the SQL statement to indicate +** the column. +** +** If there is no match, return 0. Return -1 if an error occurs. +*/ +static int resolveOrderByTermToExprList( + Parse *pParse, /* Parsing context for error messages */ + Select *pSelect, /* The SELECT statement with the ORDER BY clause */ + Expr *pE /* The specific ORDER BY term */ +){ + int i; /* Loop counter */ + ExprList *pEList; /* The columns of the result set */ + NameContext nc; /* Name context for resolving pE */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code from subprocedures */ + u8 savedSuppErr; /* Saved value of db->suppressErr */ + + assert( sqlite3ExprIsInteger(pE, &i)==0 ); + pEList = pSelect->pEList; + + /* Resolve all names in the ORDER BY term expression + */ + memset(&nc, 0, sizeof(nc)); + nc.pParse = pParse; + nc.pSrcList = pSelect->pSrc; + nc.pEList = pEList; + nc.allowAgg = 1; + nc.nErr = 0; + db = pParse->db; + savedSuppErr = db->suppressErr; + db->suppressErr = 1; + rc = sqlite3ResolveExprNames(&nc, pE); + db->suppressErr = savedSuppErr; + if( rc ) return 0; + + /* Try to match the ORDER BY expression against an expression + ** in the result set. Return an 1-based index of the matching + ** result-set entry. + */ + for(i=0; inExpr; i++){ + if( sqlite3ExprCompare(pEList->a[i].pExpr, pE)<2 ){ + return i+1; } - case TK_CONST_FUNC: - case TK_FUNCTION: { - ExprList *pFarg; /* List of function arguments */ - int nFarg; /* Number of function arguments */ - FuncDef *pDef; /* The function definition object */ - int nId; /* Length of the function name in bytes */ - const char *zId; /* The function name */ - int constMask = 0; /* Mask of function arguments that are constant */ - int i; /* Loop counter */ - u8 enc = ENC(db); /* The text encoding used by this database */ - CollSeq *pColl = 0; /* A collating sequence */ + } - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - testcase( op==TK_CONST_FUNC ); - testcase( op==TK_FUNCTION ); - if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; - } - nFarg = pFarg ? pFarg->nExpr : 0; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); - break; - } + /* If no match, return 0. */ + return 0; +} - /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evalation of - ** arguments past the first non-NULL argument. - */ - if( pDef->flags & SQLITE_FUNC_COALESCE ){ - int endCoalesce = sqlite3VdbeMakeLabel(v); - assert( nFarg>=2 ); - sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); - for(i=1; ia[i].pExpr, target); - sqlite3ExprCachePop(pParse, 1); - } - sqlite3VdbeResolveLabel(v, endCoalesce); - break; - } +/* +** Generate an ORDER BY or GROUP BY term out-of-range error. +*/ +static void resolveOutOfRangeError( + Parse *pParse, /* The error context into which to write the error */ + const char *zType, /* "ORDER" or "GROUP" */ + int i, /* The index (1-based) of the term out of range */ + int mx /* Largest permissible value of i */ +){ + sqlite3ErrorMsg(pParse, + "%r %s BY term out of range - should be " + "between 1 and %d", i, zType, mx); +} +/* +** Analyze the ORDER BY clause in a compound SELECT statement. Modify +** each term of the ORDER BY clause is a constant integer between 1 +** and N where N is the number of columns in the compound SELECT. +** +** ORDER BY terms that are already an integer between 1 and N are +** unmodified. ORDER BY terms that are integers outside the range of +** 1 through N generate an error. ORDER BY terms that are expressions +** are matched against result set expressions of compound SELECT +** beginning with the left-most SELECT and working toward the right. +** At the first match, the ORDER BY expression is transformed into +** the integer column number. +** +** Return the number of errors seen. +*/ +static int resolveCompoundOrderBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect /* The SELECT statement containing the ORDER BY */ +){ + int i; + ExprList *pOrderBy; + ExprList *pEList; + sqlite3 *db; + int moreToDo = 1; - if( pFarg ){ - r1 = sqlite3GetTempRange(pParse, nFarg); - sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); - sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ - }else{ - r1 = 0; - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Possibly overload the function if the first argument is - ** a virtual table column. - ** - ** For infix functions (LIKE, GLOB, REGEXP, and MATCH) use the - ** second argument, not the first, as the argument to test to - ** see if it is a column in a virtual table. This is done because - ** the left operand of infix functions (the operand we want to - ** control overloading) ends up as the second argument to the - ** function. The expression "A glob B" is equivalent to - ** "glob(B,A). We want to use the A in "A glob B" to test - ** for function overloading. But we use the B term in "glob(B,A)". - */ - if( nFarg>=2 && (pExpr->flags & EP_InfixFunc) ){ - pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); - }else if( nFarg>0 ){ - pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); - } + pOrderBy = pSelect->pOrderBy; + if( pOrderBy==0 ) return 0; + db = pParse->db; +#if SQLITE_MAX_COLUMN + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + return 1; + } #endif - for(i=0; ia[i].pExpr) ){ - constMask |= (1<nExpr; i++){ + pOrderBy->a[i].done = 0; + } + pSelect->pNext = 0; + while( pSelect->pPrior ){ + pSelect->pPrior->pNext = pSelect; + pSelect = pSelect->pPrior; + } + while( pSelect && moreToDo ){ + struct ExprList_item *pItem; + moreToDo = 0; + pEList = pSelect->pEList; + assert( pEList!=0 ); + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + int iCol = -1; + Expr *pE, *pDup; + if( pItem->done ) continue; + pE = pItem->pExpr; + if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( iCol<=0 || iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + return 1; } - if( (pDef->flags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ - pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); + }else{ + iCol = resolveAsName(pParse, pEList, pE); + if( iCol==0 ){ + pDup = sqlite3ExprDup(db, pE, 0); + if( !db->mallocFailed ){ + assert(pDup); + iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); + } + sqlite3ExprDelete(db, pDup); } } - if( pDef->flags & SQLITE_FUNC_NEEDCOLL ){ - if( !pColl ) pColl = db->pDfltColl; - sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); - } - sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, - (char*)pDef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nFarg); - if( nFarg ){ - sqlite3ReleaseTempRange(pParse, r1, nFarg); + if( iCol>0 ){ + CollSeq *pColl = pE->pColl; + int flags = pE->flags & EP_ExpCollate; + sqlite3ExprDelete(db, pE); + pItem->pExpr = pE = sqlite3Expr(db, TK_INTEGER, 0); + if( pE==0 ) return 1; + pE->pColl = pColl; + pE->flags |= EP_IntValue | flags; + pE->u.iValue = iCol; + pItem->iCol = (u16)iCol; + pItem->done = 1; + }else{ + moreToDo = 1; } - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_EXISTS: - case TK_SELECT: { - testcase( op==TK_EXISTS ); - testcase( op==TK_SELECT ); - inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); - break; } - case TK_IN: { - int destIfFalse = sqlite3VdbeMakeLabel(v); - int destIfNull = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Null, 0, target); - sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeAddOp2(v, OP_Integer, 1, target); - sqlite3VdbeResolveLabel(v, destIfFalse); - sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); - sqlite3VdbeResolveLabel(v, destIfNull); - break; + pSelect = pSelect->pNext; + } + for(i=0; inExpr; i++){ + if( pOrderBy->a[i].done==0 ){ + sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " + "column in the result set", i+1); + return 1; } -#endif /* SQLITE_OMIT_SUBQUERY */ - + } + return 0; +} - /* - ** x BETWEEN y AND z - ** - ** This is equivalent to - ** - ** x>=y AND x<=z - ** - ** X is stored in pExpr->pLeft. - ** Y is stored in pExpr->pList->a[0].pExpr. - ** Z is stored in pExpr->pList->a[1].pExpr. - */ - case TK_BETWEEN: { - Expr *pLeft = pExpr->pLeft; - struct ExprList_item *pLItem = pExpr->x.pList->a; - Expr *pRight = pLItem->pExpr; +/* +** Check every term in the ORDER BY or GROUP BY clause pOrderBy of +** the SELECT statement pSelect. If any term is reference to a +** result set expression (as determined by the ExprList.a.iCol field) +** then convert that term into a copy of the corresponding result set +** column. +** +** If any errors are detected, add an error message to pParse and +** return non-zero. Return zero if no errors are seen. +*/ +SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect, /* The SELECT statement containing the clause */ + ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ + const char *zType /* "ORDER" or "GROUP" */ +){ + int i; + sqlite3 *db = pParse->db; + ExprList *pEList; + struct ExprList_item *pItem; - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - r3 = sqlite3GetTempReg(pParse); - r4 = sqlite3GetTempReg(pParse); - codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); - pLItem++; - pRight = pLItem->pExpr; - sqlite3ReleaseTempReg(pParse, regFree2); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree2==0 ); - codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); - sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); - sqlite3ReleaseTempReg(pParse, r3); - sqlite3ReleaseTempReg(pParse, r4); - break; + if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; +#if SQLITE_MAX_COLUMN + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); + return 1; + } +#endif + pEList = pSelect->pEList; + assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + if( pItem->iCol ){ + if( pItem->iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + return 1; + } + resolveAlias(pParse, pEList, pItem->iCol-1, pItem->pExpr, zType); } - case TK_UPLUS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; + } + return 0; +} + +/* +** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. +** The Name context of the SELECT statement is pNC. zType is either +** "ORDER" or "GROUP" depending on which type of clause pOrderBy is. +** +** This routine resolves each term of the clause into an expression. +** If the order-by term is an integer I between 1 and N (where N is the +** number of columns in the result set of the SELECT) then the expression +** in the resolution is a copy of the I-th result-set expression. If +** the order-by term is an identify that corresponds to the AS-name of +** a result-set expression, then the term resolves to a copy of the +** result-set expression. Otherwise, the expression is resolved in +** the usual way - using sqlite3ResolveExprNames(). +** +** This routine returns the number of errors. If errors occur, then +** an appropriate error message might be left in pParse. (OOM errors +** excepted.) +*/ +static int resolveOrderGroupBy( + NameContext *pNC, /* The name context of the SELECT statement */ + Select *pSelect, /* The SELECT statement holding pOrderBy */ + ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ + const char *zType /* Either "ORDER" or "GROUP", as appropriate */ +){ + int i; /* Loop counter */ + int iCol; /* Column number */ + struct ExprList_item *pItem; /* A term of the ORDER BY clause */ + Parse *pParse; /* Parsing context */ + int nResult; /* Number of terms in the result set */ + + if( pOrderBy==0 ) return 0; + nResult = pSelect->pEList->nExpr; + pParse = pNC->pParse; + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + Expr *pE = pItem->pExpr; + iCol = resolveAsName(pParse, pSelect->pEList, pE); + if( iCol>0 ){ + /* If an AS-name match is found, mark this ORDER BY column as being + ** a copy of the iCol-th result-set column. The subsequent call to + ** sqlite3ResolveOrderGroupBy() will convert the expression to a + ** copy of the iCol-th result-set expression. */ + pItem->iCol = (u16)iCol; + continue; + } + if( sqlite3ExprIsInteger(pE, &iCol) ){ + /* The ORDER BY term is an integer constant. Again, set the column + ** number so that sqlite3ResolveOrderGroupBy() will convert the + ** order-by term to a copy of the result-set expression */ + if( iCol<1 ){ + resolveOutOfRangeError(pParse, zType, i+1, nResult); + return 1; + } + pItem->iCol = (u16)iCol; + continue; } - case TK_TRIGGER: { - /* If the opcode is TK_TRIGGER, then the expression is a reference - ** to a column in the new.* or old.* pseudo-tables available to - ** trigger programs. In this case Expr.iTable is set to 1 for the - ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn - ** is set to the column of the pseudo-table to read, or to -1 to - ** read the rowid field. - ** - ** The expression is implemented using an OP_Param opcode. The p1 - ** parameter is set to 0 for an old.rowid reference, or to (i+1) - ** to reference another column of the old.* pseudo-table, where - ** i is the index of the column. For a new.rowid reference, p1 is - ** set to (n+1), where n is the number of columns in each pseudo-table. - ** For a reference to any other column in the new.* pseudo-table, p1 - ** is set to (n+2+i), where n and i are as defined previously. For - ** example, if the table on which triggers are being fired is - ** declared as: - ** - ** CREATE TABLE t1(a, b); - ** - ** Then p1 is interpreted as follows: - ** - ** p1==0 -> old.rowid p1==3 -> new.rowid - ** p1==1 -> old.a p1==4 -> new.a - ** p1==2 -> old.b p1==5 -> new.b - */ - Table *pTab = pExpr->pTab; - int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + pExpr->iColumn; + /* Otherwise, treat the ORDER BY term as an ordinary expression */ + pItem->iCol = 0; + if( sqlite3ResolveExprNames(pNC, pE) ){ + return 1; + } + } + return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); +} - assert( pExpr->iTable==0 || pExpr->iTable==1 ); - assert( pExpr->iColumn>=-1 && pExpr->iColumnnCol ); - assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey ); - assert( p1>=0 && p1<(pTab->nCol*2+2) ); +/* +** Resolve names in the SELECT statement p and all of its descendents. +*/ +static int resolveSelectStep(Walker *pWalker, Select *p){ + NameContext *pOuterNC; /* Context that contains this SELECT */ + NameContext sNC; /* Name context of this SELECT */ + int isCompound; /* True if p is a compound select */ + int nCompound; /* Number of compound terms processed so far */ + Parse *pParse; /* Parsing context */ + ExprList *pEList; /* Result set expression list */ + int i; /* Loop counter */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Select *pLeftmost; /* Left-most of SELECT of a compound */ + sqlite3 *db; /* Database connection */ + - sqlite3VdbeAddOp2(v, OP_Param, p1, target); - VdbeComment((v, "%s.%s -> $%d", - (pExpr->iTable ? "new" : "old"), - (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName), - target - )); + assert( p!=0 ); + if( p->selFlags & SF_Resolved ){ + return WRC_Prune; + } + pOuterNC = pWalker->u.pNC; + pParse = pWalker->pParse; + db = pParse->db; -#ifndef SQLITE_OMIT_FLOATING_POINT - /* If the column has REAL affinity, it may currently be stored as an - ** integer. Use OP_RealAffinity to make sure it is really real. */ - if( pExpr->iColumn>=0 - && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL - ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, target); - } -#endif - break; - } + /* Normally sqlite3SelectExpand() will be called first and will have + ** already expanded this SELECT. However, if this is a subquery within + ** an expression, sqlite3ResolveExprNames() will be called without a + ** prior call to sqlite3SelectExpand(). When that happens, let + ** sqlite3SelectPrep() do all of the processing for this SELECT. + ** sqlite3SelectPrep() will invoke both sqlite3SelectExpand() and + ** this routine in the correct order. + */ + if( (p->selFlags & SF_Expanded)==0 ){ + sqlite3SelectPrep(pParse, p, pOuterNC); + return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + } + isCompound = p->pPrior!=0; + nCompound = 0; + pLeftmost = p; + while( p ){ + assert( (p->selFlags & SF_Expanded)!=0 ); + assert( (p->selFlags & SF_Resolved)==0 ); + p->selFlags |= SF_Resolved; - /* - ** Form A: - ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END - ** - ** Form B: - ** CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END - ** - ** Form A is can be transformed into the equivalent form B as follows: - ** CASE WHEN x=e1 THEN r1 WHEN x=e2 THEN r2 ... - ** WHEN x=eN THEN rN ELSE y END - ** - ** X (if it exists) is in pExpr->pLeft. - ** Y is in pExpr->pRight. The Y is also optional. If there is no - ** ELSE clause and no other term matches, then the result of the - ** exprssion is NULL. - ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. + /* Resolve the expressions in the LIMIT and OFFSET clauses. These + ** are not allowed to refer to any names, so pass an empty NameContext. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + if( sqlite3ResolveExprNames(&sNC, p->pLimit) || + sqlite3ResolveExprNames(&sNC, p->pOffset) ){ + return WRC_Abort; + } + + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.allowAgg = 1; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; inExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } + + /* Recursively resolve names in all subqueries + */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->pSelect ){ + const char *zSavedContext = pParse->zAuthContext; + if( pItem->zName ) pParse->zAuthContext = pItem->zName; + sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + pParse->zAuthContext = zSavedContext; + if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + } + } + + /* If there are no aggregate functions in the result-set, and no GROUP BY + ** expression, do not allow aggregates in any of the other expressions. + */ + assert( (p->selFlags & SF_Aggregate)==0 ); + pGroupBy = p->pGroupBy; + if( pGroupBy || sNC.hasAgg ){ + p->selFlags |= SF_Aggregate; + }else{ + sNC.allowAgg = 0; + } + + /* If a HAVING clause is present, then there must be a GROUP BY clause. + */ + if( p->pHaving && !pGroupBy ){ + sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); + return WRC_Abort; + } + + /* Add the expression list to the name-context before parsing the + ** other expressions in the SELECT statement. This is so that + ** expressions in the WHERE clause (etc.) can refer to expressions by + ** aliases in the result set. ** - ** The result of the expression is the Ri for the first matching Ei, - ** or if there is no matching Ei, the ELSE term Y, or if there is - ** no ELSE term, NULL. + ** Minor point: If this is the case, then the expression will be + ** re-evaluated for each reference to it. */ - default: assert( op==TK_CASE ); { - int endLabel; /* GOTO label for end of CASE stmt */ - int nextCase; /* GOTO label for next WHEN clause */ - int nExpr; /* 2x number of WHEN terms */ - int i; /* Loop counter */ - ExprList *pEList; /* List of WHEN terms */ - struct ExprList_item *aListelem; /* Array of WHEN terms */ - Expr opCompare; /* The X==Ei expression */ - Expr cacheX; /* Cached expression X */ - Expr *pX; /* The X expression */ - Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ - VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) + sNC.pEList = p->pEList; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) || + sqlite3ResolveExprNames(&sNC, p->pHaving) + ){ + return WRC_Abort; + } - assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); - assert((pExpr->x.pList->nExpr % 2) == 0); - assert(pExpr->x.pList->nExpr > 0); - pEList = pExpr->x.pList; - aListelem = pEList->a; - nExpr = pEList->nExpr; - endLabel = sqlite3VdbeMakeLabel(v); - if( (pX = pExpr->pLeft)!=0 ){ - cacheX = *pX; - testcase( pX->op==TK_COLUMN ); - testcase( pX->op==TK_REGISTER ); - cacheX.iTable = sqlite3ExprCodeTemp(pParse, pX, ®Free1); - testcase( regFree1==0 ); - cacheX.op = TK_REGISTER; - opCompare.op = TK_EQ; - opCompare.pLeft = &cacheX; - pTest = &opCompare; - /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: - ** The value in regFree1 might get SCopy-ed into the file result. - ** So make sure that the regFree1 register is not reused for other - ** purposes and possibly overwritten. */ - regFree1 = 0; + /* The ORDER BY and GROUP BY clauses may not refer to terms in + ** outer queries + */ + sNC.pNext = 0; + sNC.allowAgg = 1; + + /* Process the ORDER BY clause for singleton SELECT statements. + ** The ORDER BY clause for compounds SELECT statements is handled + ** below, after all of the result-sets for all of the elements of + ** the compound have been resolved. + */ + if( !isCompound && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ + return WRC_Abort; + } + if( db->mallocFailed ){ + return WRC_Abort; + } + + /* Resolve the GROUP BY clause. At the same time, make sure + ** the GROUP BY clause does not contain aggregate functions. + */ + if( pGroupBy ){ + struct ExprList_item *pItem; + + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ + return WRC_Abort; } - for(i=0; ia; inExpr; i++, pItem++){ + if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " + "the GROUP BY clause"); + return WRC_Abort; } - nextCase = sqlite3VdbeMakeLabel(v); - testcase( pTest->op==TK_COLUMN ); - sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); - testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); - testcase( aListelem[i+1].pExpr->op==TK_REGISTER ); - sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); - sqlite3ExprCachePop(pParse, 1); - sqlite3VdbeResolveLabel(v, nextCase); } - if( pExpr->pRight ){ - sqlite3ExprCachePush(pParse); - sqlite3ExprCode(pParse, pExpr->pRight, target); - sqlite3ExprCachePop(pParse, 1); - }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, target); - } - assert( db->mallocFailed || pParse->nErr>0 - || pParse->iCacheLevel==iCacheLevel ); - sqlite3VdbeResolveLabel(v, endLabel); - break; } -#ifndef SQLITE_OMIT_TRIGGER - case TK_RAISE: { - assert( pExpr->affinity==OE_Rollback - || pExpr->affinity==OE_Abort - || pExpr->affinity==OE_Fail - || pExpr->affinity==OE_Ignore - ); - if( !pParse->pTriggerTab ){ - sqlite3ErrorMsg(pParse, - "RAISE() may only be used within a trigger-program"); - return 0; - } - if( pExpr->affinity==OE_Abort ){ - sqlite3MayAbort(pParse); - } - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->affinity==OE_Ignore ){ - sqlite3VdbeAddOp4( - v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); - }else{ - sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); - } - break; - } -#endif + /* Advance to the next term of the compound + */ + p = p->pPrior; + nCompound++; } - sqlite3ReleaseTempReg(pParse, regFree1); - sqlite3ReleaseTempReg(pParse, regFree2); - return inReg; -} -/* -** Generate code to evaluate an expression and store the results -** into a register. Return the register number where the results -** are stored. -** -** If the register is a temporary register that can be deallocated, -** then write its number into *pReg. If the result register is not -** a temporary, then set *pReg to zero. -*/ -SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ - int r1 = sqlite3GetTempReg(pParse); - int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( r2==r1 ){ - *pReg = r1; - }else{ - sqlite3ReleaseTempReg(pParse, r1); - *pReg = 0; + /* Resolve the ORDER BY on a compound SELECT after all terms of + ** the compound have been resolved. + */ + if( isCompound && resolveCompoundOrderBy(pParse, pLeftmost) ){ + return WRC_Abort; } - return r2; + + return WRC_Prune; } /* -** Generate code that will evaluate expression pExpr and store the -** results in register target. The results are guaranteed to appear -** in register target. +** This routine walks an expression tree and resolves references to +** table columns and result-set columns. At the same time, do error +** checking on function usage and set a flag if any aggregate functions +** are seen. +** +** To resolve table columns references we look for nodes (or subtrees) of the +** form X.Y.Z or Y.Z or just Z where +** +** X: The name of a database. Ex: "main" or "temp" or +** the symbolic name assigned to an ATTACH-ed database. +** +** Y: The name of a table in a FROM clause. Or in a trigger +** one of the special names "old" or "new". +** +** Z: The name of a column in table Y. +** +** The node at the root of the subtree is modified as follows: +** +** Expr.op Changed to TK_COLUMN +** Expr.pTab Points to the Table object for X.Y +** Expr.iColumn The column index in X.Y. -1 for the rowid. +** Expr.iTable The VDBE cursor number for X.Y +** +** +** To resolve result-set references, look for expression nodes of the +** form Z (with no X and Y prefix) where the Z matches the right-hand +** size of an AS clause in the result-set of a SELECT. The Z expression +** is replaced by a copy of the left-hand side of the result-set expression. +** Table-name and function resolution occurs on the substituted expression +** tree. For example, in: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY x; +** +** The "x" term of the order by is replaced by "a+b" to render: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; +** +** Function calls are checked to make sure that the function is +** defined and that the correct number of arguments are specified. +** If the function is an aggregate function, then the pNC->hasAgg is +** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. +** If an expression contains aggregate functions then the EP_Agg +** property on the expression is set. +** +** An error message is left in pParse if anything is amiss. The number +** if errors is returned. */ -SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ - int inReg; +SQLITE_PRIVATE int sqlite3ResolveExprNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + Expr *pExpr /* The expression to be analyzed. */ +){ + int savedHasAgg; + Walker w; - assert( target>0 && target<=pParse->nMem ); - if( pExpr && pExpr->op==TK_REGISTER ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); - }else{ - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + if( pExpr==0 ) return 0; +#if SQLITE_MAX_EXPR_DEPTH>0 + { + Parse *pParse = pNC->pParse; + if( sqlite3ExprCheckHeight(pParse, pExpr->nHeight+pNC->pParse->nHeight) ){ + return 1; } + pParse->nHeight += pExpr->nHeight; } - return target; +#endif + savedHasAgg = pNC->hasAgg; + pNC->hasAgg = 0; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.pParse = pNC->pParse; + w.u.pNC = pNC; + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + pNC->pParse->nHeight -= pExpr->nHeight; +#endif + if( pNC->nErr>0 || w.pParse->nErr>0 ){ + ExprSetProperty(pExpr, EP_Error); + } + if( pNC->hasAgg ){ + ExprSetProperty(pExpr, EP_Agg); + }else if( savedHasAgg ){ + pNC->hasAgg = 1; + } + return ExprHasProperty(pExpr, EP_Error); } + /* -** Generate code that evalutes the given expression and puts the result -** in register target. +** Resolve all names in all expressions of a SELECT and in all +** decendents of the SELECT, including compounds off of p->pPrior, +** subqueries in expressions, and subqueries used as FROM clause +** terms. ** -** Also make a copy of the expression results into another "cache" register -** and modify the expression so that the next time it is evaluated, -** the result is a copy of the cache register. +** See sqlite3ResolveExprNames() for a description of the kinds of +** transformations that occur. ** -** This routine is used for expressions that are used multiple -** times. They are evaluated once and the results of the expression -** are reused. +** All SELECT statements should have been expanded using +** sqlite3SelectExpand() prior to invoking this routine. */ -SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ - Vdbe *v = pParse->pVdbe; - int inReg; - inReg = sqlite3ExprCode(pParse, pExpr, target); - assert( target>0 ); - /* This routine is called for terms to INSERT or UPDATE. And the only - ** other place where expressions can be converted into TK_REGISTER is - ** in WHERE clause processing. So as currently implemented, there is - ** no way for a TK_REGISTER to exist here. But it seems prudent to - ** keep the ALWAYS() in case the conditions above change with future - ** modifications or enhancements. */ - if( ALWAYS(pExpr->op!=TK_REGISTER) ){ - int iMem; - iMem = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, inReg, iMem); - pExpr->iTable = iMem; - pExpr->op2 = pExpr->op; - pExpr->op = TK_REGISTER; - } - return inReg; +SQLITE_PRIVATE void sqlite3ResolveSelectNames( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for parent SELECT statement */ +){ + Walker w; + + assert( p!=0 ); + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.pParse = pParse; + w.u.pNC = pOuterNC; + sqlite3WalkSelect(&w, p); } +/************** End of resolve.c *********************************************/ +/************** Begin file expr.c ********************************************/ /* -** Return TRUE if pExpr is an constant expression that is appropriate -** for factoring out of a loop. Appropriate expressions are: +** 2001 September 15 ** -** * Any expression that evaluates to two or more opcodes. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** * Any OP_Integer, OP_Real, OP_String, OP_Blob, OP_Null, -** or OP_Variable that does not need to be placed in a -** specific register. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** There is no point in factoring out single-instruction constant -** expressions that need to be placed in a particular register. -** We could factor them out, but then we would end up adding an -** OP_SCopy instruction to move the value into the correct register -** later. We might as well just use the original instruction and -** avoid the OP_SCopy. +************************************************************************* +** This file contains routines used for analyzing expressions and +** for generating VDBE code that evaluates expressions in SQLite. */ -static int isAppropriateForFactoring(Expr *p){ - if( !sqlite3ExprIsConstantNotJoin(p) ){ - return 0; /* Only constant expressions are appropriate for factoring */ + +/* +** Return the 'affinity' of the expression pExpr if any. +** +** If pExpr is a column, a reference to a column via an 'AS' alias, +** or a sub-select with a column as the return value, then the +** affinity of that column is returned. Otherwise, 0x00 is returned, +** indicating no affinity for the expression. +** +** i.e. the WHERE clause expresssions in the following statements all +** have an affinity: +** +** CREATE TABLE t1(a); +** SELECT * FROM t1 WHERE a; +** SELECT a AS b FROM t1 WHERE b; +** SELECT * FROM t1 WHERE (select a from t1); +*/ +SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ + int op = pExpr->op; + if( op==TK_SELECT ){ + assert( pExpr->flags&EP_xIsSelect ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); } - if( (p->flags & EP_FixedDest)==0 ){ - return 1; /* Any constant without a fixed destination is appropriate */ +#ifndef SQLITE_OMIT_CAST + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken); } - while( p->op==TK_UPLUS ) p = p->pLeft; - switch( p->op ){ -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: #endif - case TK_VARIABLE: - case TK_INTEGER: - case TK_FLOAT: - case TK_NULL: - case TK_STRING: { - testcase( p->op==TK_BLOB ); - testcase( p->op==TK_VARIABLE ); - testcase( p->op==TK_INTEGER ); - testcase( p->op==TK_FLOAT ); - testcase( p->op==TK_NULL ); - testcase( p->op==TK_STRING ); - /* Single-instruction constants with a fixed destination are - ** better done in-line. If we factor them, they will just end - ** up generating an OP_SCopy to move the value to the destination - ** register. */ - return 0; - } - case TK_UMINUS: { - if( p->pLeft->op==TK_FLOAT || p->pLeft->op==TK_INTEGER ){ - return 0; + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) + && pExpr->pTab!=0 + ){ + /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = pExpr->iColumn; + if( j<0 ) return SQLITE_AFF_INTEGER; + assert( pExpr->pTab && jpTab->nCol ); + return pExpr->pTab->aCol[j].affinity; + } + return pExpr->affinity; +} + +/* +** Set the explicit collating sequence for an expression to the +** collating sequence supplied in the second argument. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ + if( pExpr && pColl ){ + pExpr->pColl = pColl; + pExpr->flags |= EP_ExpCollate; + } + return pExpr; +} + +/* +** Set the collating sequence for expression pExpr to be the collating +** sequence named by pToken. Return a pointer to the revised expression. +** The collating sequence is marked as "explicit" using the EP_ExpCollate +** flag. An explicit collating sequence will override implicit +** collating sequences. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ + char *zColl = 0; /* Dequoted name of collation sequence */ + CollSeq *pColl; + sqlite3 *db = pParse->db; + zColl = sqlite3NameFromToken(db, pCollName); + pColl = sqlite3LocateCollSeq(pParse, zColl); + sqlite3ExprSetColl(pExpr, pColl); + sqlite3DbFree(db, zColl); + return pExpr; +} + +/* +** Return the default collation sequence for the expression pExpr. If +** there is no default collation type, return 0. +*/ +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ + CollSeq *pColl = 0; + Expr *p = pExpr; + while( p ){ + int op; + pColl = p->pColl; + if( pColl ) break; + op = p->op; + if( p->pTab!=0 && ( + op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER + )){ + /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + const char *zColl; + int j = p->iColumn; + if( j>=0 ){ + sqlite3 *db = pParse->db; + zColl = p->pTab->aCol[j].zColl; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + pExpr->pColl = pColl; } break; } - default: { + if( op!=TK_CAST && op!=TK_UPLUS ){ break; } + p = p->pLeft; } - return 1; + if( sqlite3CheckCollSeq(pParse, pColl) ){ + pColl = 0; + } + return pColl; } /* -** If pExpr is a constant expression that is appropriate for -** factoring out of a loop, then evaluate the expression -** into a register and convert the expression into a TK_REGISTER -** expression. +** pExpr is an operand of a comparison operator. aff2 is the +** type affinity of the other operand. This routine returns the +** type affinity that should be used for the comparison operator. */ -static int evalConstExpr(Walker *pWalker, Expr *pExpr){ - Parse *pParse = pWalker->pParse; - switch( pExpr->op ){ - case TK_IN: - case TK_REGISTER: { - return WRC_Prune; - } - case TK_FUNCTION: - case TK_AGG_FUNCTION: - case TK_CONST_FUNC: { - /* The arguments to a function have a fixed destination. - ** Mark them this way to avoid generated unneeded OP_SCopy - ** instructions. - */ - ExprList *pList = pExpr->x.pList; - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - if( pList ){ - int i = pList->nExpr; - struct ExprList_item *pItem = pList->a; - for(; i>0; i--, pItem++){ - if( ALWAYS(pItem->pExpr) ) pItem->pExpr->flags |= EP_FixedDest; - } - } - break; +SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ + char aff1 = sqlite3ExprAffinity(pExpr); + if( aff1 && aff2 ){ + /* Both sides of the comparison are columns. If one has numeric + ** affinity, use that. Otherwise use no affinity. + */ + if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ + return SQLITE_AFF_NUMERIC; + }else{ + return SQLITE_AFF_NONE; } + }else if( !aff1 && !aff2 ){ + /* Neither side of the comparison is a column. Compare the + ** results directly. + */ + return SQLITE_AFF_NONE; + }else{ + /* One side is a column, the other is not. Use the columns affinity. */ + assert( aff1==0 || aff2==0 ); + return (aff1 + aff2); } - if( isAppropriateForFactoring(pExpr) ){ - int r1 = ++pParse->nMem; - int r2; - r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); - if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); - pExpr->op2 = pExpr->op; - pExpr->op = TK_REGISTER; - pExpr->iTable = r2; - return WRC_Prune; +} + +/* +** pExpr is a comparison operator. Return the type affinity that should +** be applied to both operands prior to doing the comparison. +*/ +static char comparisonAffinity(Expr *pExpr){ + char aff; + assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || + pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || + pExpr->op==TK_NE || pExpr->op==TK_IS || pExpr->op==TK_ISNOT ); + assert( pExpr->pLeft ); + aff = sqlite3ExprAffinity(pExpr->pLeft); + if( pExpr->pRight ){ + aff = sqlite3CompareAffinity(pExpr->pRight, aff); + }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); + }else if( !aff ){ + aff = SQLITE_AFF_NONE; } - return WRC_Continue; + return aff; } /* -** Preevaluate constant subexpressions within pExpr and store the -** results in registers. Modify pExpr so that the constant subexpresions -** are TK_REGISTER opcodes that refer to the precomputed values. -** -** This routine is a no-op if the jump to the cookie-check code has -** already occur. Since the cookie-check jump is generated prior to -** any other serious processing, this check ensures that there is no -** way to accidently bypass the constant initializations. -** -** This routine is also a no-op if the SQLITE_FactorOutConst optimization -** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) -** interface. This allows test logic to verify that the same answer is -** obtained for queries regardless of whether or not constants are -** precomputed into registers or if they are inserted in-line. +** pExpr is a comparison expression, eg. '=', '<', IN(...) etc. +** idx_affinity is the affinity of an indexed column. Return true +** if the index with affinity idx_affinity may be used to implement +** the comparison in pExpr. */ -SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ - Walker w; - if( pParse->cookieGoto ) return; - if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; - w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; - w.pParse = pParse; - sqlite3WalkExpr(&w, pExpr); +SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ + char aff = comparisonAffinity(pExpr); + switch( aff ){ + case SQLITE_AFF_NONE: + return 1; + case SQLITE_AFF_TEXT: + return idx_affinity==SQLITE_AFF_TEXT; + default: + return sqlite3IsNumericAffinity(idx_affinity); + } } +/* +** Return the P5 value that should be used for a binary comparison +** opcode (OP_Eq, OP_Ge etc.) used to compare pExpr1 and pExpr2. +*/ +static u8 binaryCompareP5(Expr *pExpr1, Expr *pExpr2, int jumpIfNull){ + u8 aff = (char)sqlite3ExprAffinity(pExpr2); + aff = (u8)sqlite3CompareAffinity(pExpr1, aff) | (u8)jumpIfNull; + return aff; +} /* -** Generate code that pushes the value of every element of the given -** expression list into a sequence of registers beginning at target. +** Return a pointer to the collation sequence that should be used by +** a binary comparison operator comparing pLeft and pRight. ** -** Return the number of elements evaluated. +** If the left hand expression has a collating sequence type, then it is +** used. Otherwise the collation sequence for the right hand expression +** is used, or the default (BINARY) if neither expression has a collating +** type. +** +** Argument pRight (but not pLeft) may be a null pointer. In this case, +** it is not considered. */ -SQLITE_PRIVATE int sqlite3ExprCodeExprList( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* The expression list to be coded */ - int target, /* Where to write results */ - int doHardCopy /* Make a hard copy of every element */ +SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq( + Parse *pParse, + Expr *pLeft, + Expr *pRight ){ - struct ExprList_item *pItem; - int i, n; - assert( pList!=0 ); - assert( target>0 ); - assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ - n = pList->nExpr; - for(pItem=pList->a, i=0; ipExpr; - int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); - if( inReg!=target+i ){ - sqlite3VdbeAddOp2(pParse->pVdbe, doHardCopy ? OP_Copy : OP_SCopy, - inReg, target+i); + CollSeq *pColl; + assert( pLeft ); + if( pLeft->flags & EP_ExpCollate ){ + assert( pLeft->pColl ); + pColl = pLeft->pColl; + }else if( pRight && pRight->flags & EP_ExpCollate ){ + assert( pRight->pColl ); + pColl = pRight->pColl; + }else{ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + if( !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); } } - return n; + return pColl; } /* -** Generate code for a BETWEEN operator. -** -** x BETWEEN y AND z -** -** The above is equivalent to -** -** x>=y AND x<=z -** -** Code it as such, taking care to do the common subexpression -** elementation of x. +** Generate code for a comparison operator. */ -static void exprCodeBetween( - Parse *pParse, /* Parsing and code generating context */ - Expr *pExpr, /* The BETWEEN expression */ - int dest, /* Jump here if the jump is taken */ - int jumpIfTrue, /* Take the jump if the BETWEEN is true */ - int jumpIfNull /* Take the jump if the BETWEEN is NULL */ +static int codeCompare( + Parse *pParse, /* The parsing (and code generating) context */ + Expr *pLeft, /* The left operand */ + Expr *pRight, /* The right operand */ + int opcode, /* The comparison opcode */ + int in1, int in2, /* Register holding operands */ + int dest, /* Jump here if true. */ + int jumpIfNull /* If true, jump if either operand is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ - Expr compLeft; /* The x>=y term */ - Expr compRight; /* The x<=z term */ - Expr exprX; /* The x subexpression */ - int regFree1 = 0; /* Temporary use register */ - - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - exprX = *pExpr->pLeft; - exprAnd.op = TK_AND; - exprAnd.pLeft = &compLeft; - exprAnd.pRight = &compRight; - compLeft.op = TK_GE; - compLeft.pLeft = &exprX; - compLeft.pRight = pExpr->x.pList->a[0].pExpr; - compRight.op = TK_LE; - compRight.pLeft = &exprX; - compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); - exprX.op = TK_REGISTER; - if( jumpIfTrue ){ - sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); - }else{ - sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); - } - sqlite3ReleaseTempReg(pParse, regFree1); + int p5; + int addr; + CollSeq *p4; - /* Ensure adequate test coverage */ - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); + addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, + (void*)p4, P4_COLLSEQ); + sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + return addr; } +#if SQLITE_MAX_EXPR_DEPTH>0 /* -** Generate code for a boolean expression such that a jump is made -** to the label "dest" if the expression is true but execution -** continues straight thru if the expression is false. -** -** If the expression evaluates to NULL (neither true nor false), then -** take the jump if the jumpIfNull flag is SQLITE_JUMPIFNULL. -** -** This code depends on the fact that certain token values (ex: TK_EQ) -** are the same as opcode values (ex: OP_Eq) that implement the corresponding -** operation. Special comments in vdbe.c and the mkopcodeh.awk script in -** the make process cause these values to align. Assert()s in the code -** below verify that the numbers are aligned correctly. +** Check that argument nHeight is less than or equal to the maximum +** expression depth allowed. If it is not, leave an error message in +** pParse. */ -SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ - Vdbe *v = pParse->pVdbe; - int op = 0; - int regFree1 = 0; - int regFree2 = 0; - int r1, r2; +SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ + int rc = SQLITE_OK; + int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; + if( nHeight>mxHeight ){ + sqlite3ErrorMsg(pParse, + "Expression tree is too large (maximum depth %d)", mxHeight + ); + rc = SQLITE_ERROR; + } + return rc; +} - assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ - if( NEVER(pExpr==0) ) return; /* No way this can happen */ - op = pExpr->op; - switch( op ){ - case TK_AND: { - int d2 = sqlite3VdbeMakeLabel(v); - testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); - break; - } - case TK_OR: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - break; +/* The following three functions, heightOfExpr(), heightOfExprList() +** and heightOfSelect(), are used to determine the maximum height +** of any expression tree referenced by the structure passed as the +** first argument. +** +** If this maximum height is greater than the current value pointed +** to by pnHeight, the second parameter, then set *pnHeight to that +** value. +*/ +static void heightOfExpr(Expr *p, int *pnHeight){ + if( p ){ + if( p->nHeight>*pnHeight ){ + *pnHeight = p->nHeight; } - case TK_NOT: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - break; - } - case TK_LT: - case TK_LE: - case TK_GT: - case TK_GE: - case TK_NE: - case TK_EQ: { - assert( TK_LT==OP_Lt ); - assert( TK_LE==OP_Le ); - assert( TK_GT==OP_Gt ); - assert( TK_GE==OP_Ge ); - assert( TK_EQ==OP_Eq ); - assert( TK_NE==OP_Ne ); - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, jumpIfNull); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_ISNULL: - case TK_NOTNULL: { - assert( TK_ISNULL==OP_IsNull ); - assert( TK_NOTNULL==OP_NotNull ); - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeAddOp2(v, op, r1, dest); - testcase( regFree1==0 ); - break; - } - case TK_BETWEEN: { - testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); - break; - } - case TK_IN: { - int destIfFalse = sqlite3VdbeMakeLabel(v); - int destIfNull = jumpIfNull ? dest : destIfFalse; - sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); - sqlite3VdbeResolveLabel(v, destIfFalse); - break; - } - default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); - break; + } +} +static void heightOfExprList(ExprList *p, int *pnHeight){ + if( p ){ + int i; + for(i=0; inExpr; i++){ + heightOfExpr(p->a[i].pExpr, pnHeight); } } - sqlite3ReleaseTempReg(pParse, regFree1); - sqlite3ReleaseTempReg(pParse, regFree2); +} +static void heightOfSelect(Select *p, int *pnHeight){ + if( p ){ + heightOfExpr(p->pWhere, pnHeight); + heightOfExpr(p->pHaving, pnHeight); + heightOfExpr(p->pLimit, pnHeight); + heightOfExpr(p->pOffset, pnHeight); + heightOfExprList(p->pEList, pnHeight); + heightOfExprList(p->pGroupBy, pnHeight); + heightOfExprList(p->pOrderBy, pnHeight); + heightOfSelect(p->pPrior, pnHeight); + } } /* -** Generate code for a boolean expression such that a jump is made -** to the label "dest" if the expression is false but execution -** continues straight thru if the expression is true. -** -** If the expression evaluates to NULL (neither true nor false) then -** jump if jumpIfNull is SQLITE_JUMPIFNULL or fall through if jumpIfNull -** is 0. +** Set the Expr.nHeight variable in the structure passed as an +** argument. An expression with no children, Expr.pList or +** Expr.pSelect member has a height of 1. Any other expression +** has a height equal to the maximum height of any other +** referenced Expr plus one. */ -SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ - Vdbe *v = pParse->pVdbe; - int op = 0; - int regFree1 = 0; - int regFree2 = 0; - int r1, r2; +static void exprSetHeight(Expr *p){ + int nHeight = 0; + heightOfExpr(p->pLeft, &nHeight); + heightOfExpr(p->pRight, &nHeight); + if( ExprHasProperty(p, EP_xIsSelect) ){ + heightOfSelect(p->x.pSelect, &nHeight); + }else{ + heightOfExprList(p->x.pList, &nHeight); + } + p->nHeight = nHeight + 1; +} - assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ - if( pExpr==0 ) return; +/* +** Set the Expr.nHeight variable using the exprSetHeight() function. If +** the height is greater than the maximum allowed expression depth, +** leave an error in pParse. +*/ +SQLITE_PRIVATE void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ + exprSetHeight(p); + sqlite3ExprCheckHeight(pParse, p->nHeight); +} - /* The value of pExpr->op and op are related as follows: - ** - ** pExpr->op op - ** --------- ---------- - ** TK_ISNULL OP_NotNull - ** TK_NOTNULL OP_IsNull - ** TK_NE OP_Eq - ** TK_EQ OP_Ne - ** TK_GT OP_Le - ** TK_LE OP_Gt - ** TK_GE OP_Lt - ** TK_LT OP_Ge - ** - ** For other values of pExpr->op, op is undefined and unused. - ** The value of TK_ and OP_ constants are arranged such that we - ** can compute the mapping above using the following expression. - ** Assert()s verify that the computation is correct. - */ - op = ((pExpr->op+(TK_ISNULL&1))^1)-(TK_ISNULL&1); +/* +** Return the maximum height of any expression tree referenced +** by the select statement passed as an argument. +*/ +SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ + int nHeight = 0; + heightOfSelect(p, &nHeight); + return nHeight; +} +#else + #define exprSetHeight(y) +#endif /* SQLITE_MAX_EXPR_DEPTH>0 */ - /* Verify correct alignment of TK_ and OP_ constants - */ - assert( pExpr->op!=TK_ISNULL || op==OP_NotNull ); - assert( pExpr->op!=TK_NOTNULL || op==OP_IsNull ); - assert( pExpr->op!=TK_NE || op==OP_Eq ); - assert( pExpr->op!=TK_EQ || op==OP_Ne ); - assert( pExpr->op!=TK_LT || op==OP_Ge ); - assert( pExpr->op!=TK_LE || op==OP_Gt ); - assert( pExpr->op!=TK_GT || op==OP_Le ); - assert( pExpr->op!=TK_GE || op==OP_Lt ); +/* +** This routine is the core allocator for Expr nodes. +** +** Construct a new expression node and return a pointer to it. Memory +** for this node and for the pToken argument is a single allocation +** obtained from sqlite3DbMalloc(). The calling function +** is responsible for making sure the node eventually gets freed. +** +** If dequote is true, then the token (if it exists) is dequoted. +** If dequote is false, no dequoting is performance. The deQuote +** parameter is ignored if pToken is NULL or if the token does not +** appear to be quoted. If the quotes were of the form "..." (double-quotes) +** then the EP_DblQuoted flag is set on the expression node. +** +** Special case: If op==TK_INTEGER and pToken points to a string that +** can be translated into a 32-bit integer, then the token is not +** stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage +** is allocated to hold the integer text and the dequote flag is ignored. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAlloc( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + const Token *pToken, /* Token argument. Might be NULL */ + int dequote /* True to dequote */ +){ + Expr *pNew; + int nExtra = 0; + int iValue = 0; - switch( pExpr->op ){ - case TK_AND: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - break; - } - case TK_OR: { - int d2 = sqlite3VdbeMakeLabel(v); - testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); - break; - } - case TK_NOT: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - break; - } - case TK_LT: - case TK_LE: - case TK_GT: - case TK_GE: - case TK_NE: - case TK_EQ: { - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, jumpIfNull); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_ISNULL: - case TK_NOTNULL: { - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeAddOp2(v, op, r1, dest); - testcase( regFree1==0 ); - break; - } - case TK_BETWEEN: { - testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); - break; + if( pToken ){ + if( op!=TK_INTEGER || pToken->z==0 + || sqlite3GetInt32(pToken->z, &iValue)==0 ){ + nExtra = pToken->n+1; + assert( iValue>=0 ); } - case TK_IN: { - if( jumpIfNull ){ - sqlite3ExprCodeIN(pParse, pExpr, dest, dest); + } + pNew = sqlite3DbMallocZero(db, sizeof(Expr)+nExtra); + if( pNew ){ + pNew->op = (u8)op; + pNew->iAgg = -1; + if( pToken ){ + if( nExtra==0 ){ + pNew->flags |= EP_IntValue; + pNew->u.iValue = iValue; }else{ - int destIfNull = sqlite3VdbeMakeLabel(v); - sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); - sqlite3VdbeResolveLabel(v, destIfNull); + int c; + pNew->u.zToken = (char*)&pNew[1]; + memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && nExtra>=3 + && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + sqlite3Dequote(pNew->u.zToken); + if( c=='"' ) pNew->flags |= EP_DblQuoted; + } } - break; - } - default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); - break; } +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif } - sqlite3ReleaseTempReg(pParse, regFree1); - sqlite3ReleaseTempReg(pParse, regFree2); + return pNew; } /* -** Do a deep comparison of two expression trees. Return 0 if the two -** expressions are completely identical. Return 1 if they differ only -** by a COLLATE operator at the top level. Return 2 if there are differences -** other than the top-level COLLATE operator. +** Allocate a new expression node from a zero-terminated token that has +** already been dequoted. +*/ +SQLITE_PRIVATE Expr *sqlite3Expr( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + const char *zToken /* Token argument. Might be NULL */ +){ + Token x; + x.z = zToken; + x.n = zToken ? sqlite3Strlen30(zToken) : 0; + return sqlite3ExprAlloc(db, op, &x, 0); +} + +/* +** Attach subtrees pLeft and pRight to the Expr node pRoot. ** -** Sometimes this routine will return 2 even if the two expressions -** really are equivalent. If we cannot prove that the expressions are -** identical, we return 2 just to be safe. So if this routine -** returns 2, then you do not really know for certain if the two -** expressions are the same. But if you get a 0 or 1 return, then you -** can be sure the expressions are the same. In the places where -** this routine is used, it does not hurt to get an extra 2 - that -** just might result in some slightly slower code. But returning -** an incorrect 0 or 1 could lead to a malfunction. +** If pRoot==NULL that means that a memory allocation error has occurred. +** In that case, delete the subtrees pLeft and pRight. */ -SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ - if( pA==0||pB==0 ){ - return pB==pA ? 0 : 2; - } - assert( !ExprHasAnyProperty(pA, EP_TokenOnly|EP_Reduced) ); - assert( !ExprHasAnyProperty(pB, EP_TokenOnly|EP_Reduced) ); - if( ExprHasProperty(pA, EP_xIsSelect) || ExprHasProperty(pB, EP_xIsSelect) ){ - return 2; - } - if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; - if( pA->op!=pB->op ) return 2; - if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; - if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; - if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2; - if( pA->iTable!=pB->iTable || pA->iColumn!=pB->iColumn ) return 2; - if( ExprHasProperty(pA, EP_IntValue) ){ - if( !ExprHasProperty(pB, EP_IntValue) || pA->u.iValue!=pB->u.iValue ){ - return 2; +SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( + sqlite3 *db, + Expr *pRoot, + Expr *pLeft, + Expr *pRight +){ + if( pRoot==0 ){ + assert( db->mallocFailed ); + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + }else{ + if( pRight ){ + pRoot->pRight = pRight; + if( pRight->flags & EP_ExpCollate ){ + pRoot->flags |= EP_ExpCollate; + pRoot->pColl = pRight->pColl; + } } - }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ - if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; - if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return 2; + if( pLeft ){ + pRoot->pLeft = pLeft; + if( pLeft->flags & EP_ExpCollate ){ + pRoot->flags |= EP_ExpCollate; + pRoot->pColl = pLeft->pColl; + } } + exprSetHeight(pRoot); } - if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; - if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; - return 0; } /* -** Compare two ExprList objects. Return 0 if they are identical and -** non-zero if they differ in any way. -** -** This routine might return non-zero for equivalent ExprLists. The -** only consequence will be disabled optimizations. But this routine -** must never return 0 if the two ExprList objects are different, or -** a malfunction will result. +** Allocate a Expr node which joins as many as two subtrees. ** -** Two NULL pointers are considered to be the same. But a NULL pointer -** always differs from a non-NULL pointer. +** One or both of the subtrees can be NULL. Return a pointer to the new +** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, +** free the subtrees and return NULL. */ -SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB){ - int i; - if( pA==0 && pB==0 ) return 0; - if( pA==0 || pB==0 ) return 1; - if( pA->nExpr!=pB->nExpr ) return 1; - for(i=0; inExpr; i++){ - Expr *pExprA = pA->a[i].pExpr; - Expr *pExprB = pB->a[i].pExpr; - if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; - if( sqlite3ExprCompare(pExprA, pExprB) ) return 1; +SQLITE_PRIVATE Expr *sqlite3PExpr( + Parse *pParse, /* Parsing context */ + int op, /* Expression opcode */ + Expr *pLeft, /* Left operand */ + Expr *pRight, /* Right operand */ + const Token *pToken /* Argument token */ +){ + Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); + sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + if( p ) { + sqlite3ExprCheckHeight(pParse, p->nHeight); } - return 0; + return p; } /* -** Add a new element to the pAggInfo->aCol[] array. Return the index of -** the new element. Return a negative number if malloc fails. +** Join two expressions using an AND operator. If either expression is +** NULL, then just return the other expression. */ -static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ - int i; - pInfo->aCol = sqlite3ArrayAllocate( - db, - pInfo->aCol, - sizeof(pInfo->aCol[0]), - 3, - &pInfo->nColumn, - &pInfo->nColumnAlloc, - &i - ); - return i; -} +SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ + if( pLeft==0 ){ + return pRight; + }else if( pRight==0 ){ + return pLeft; + }else{ + Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); + sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); + return pNew; + } +} /* -** Add a new element to the pAggInfo->aFunc[] array. Return the index of -** the new element. Return a negative number if malloc fails. +** Construct a new expression node for a function with multiple +** arguments. */ -static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ - int i; - pInfo->aFunc = sqlite3ArrayAllocate( - db, - pInfo->aFunc, - sizeof(pInfo->aFunc[0]), - 3, - &pInfo->nFunc, - &pInfo->nFuncAlloc, - &i - ); - return i; -} +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ + Expr *pNew; + sqlite3 *db = pParse->db; + assert( pToken ); + pNew = sqlite3ExprAlloc(db, TK_FUNCTION, pToken, 1); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ + return 0; + } + pNew->x.pList = pList; + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + sqlite3ExprSetHeight(pParse, pNew); + return pNew; +} /* -** This is the xExprCallback for a tree walker. It is used to -** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates -** for additional information. +** Assign a variable number to an expression that encodes a wildcard +** in the original SQL statement. +** +** Wildcards consisting of a single "?" are assigned the next sequential +** variable number. +** +** Wildcards of the form "?nnn" are assigned the number "nnn". We make +** sure "nnn" is not too be to avoid a denial of service attack when +** the SQL statement comes from an external source. +** +** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number +** as the previous instance of the same wildcard. Or if this is the first +** instance of the wildcard, the next sequenial variable number is +** assigned. */ -static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ - int i; - NameContext *pNC = pWalker->u.pNC; - Parse *pParse = pNC->pParse; - SrcList *pSrcList = pNC->pSrcList; - AggInfo *pAggInfo = pNC->pAggInfo; +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ + sqlite3 *db = pParse->db; + const char *z; - switch( pExpr->op ){ - case TK_AGG_COLUMN: - case TK_COLUMN: { - testcase( pExpr->op==TK_AGG_COLUMN ); - testcase( pExpr->op==TK_COLUMN ); - /* Check to see if the column is in one of the tables in the FROM - ** clause of the aggregate query */ - if( ALWAYS(pSrcList!=0) ){ - struct SrcList_item *pItem = pSrcList->a; - for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; - assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); - if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable && - pCol->iColumn==pExpr->iColumn ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - pCol->pTab = pExpr->pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; - pCol->iSorterColumn = -1; - pCol->pExpr = pExpr; - if( pAggInfo->pGroupBy ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && - pE->iColumn==pExpr->iColumn ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetIrreducible(pExpr); - pExpr->pAggInfo = pAggInfo; - pExpr->op = TK_AGG_COLUMN; - pExpr->iAgg = (i16)k; - break; - } /* endif pExpr->iTable==pItem->iCursor */ - } /* end loop over pSrcList */ + if( pExpr==0 ) return; + assert( !ExprHasAnyProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); + z = pExpr->u.zToken; + assert( z!=0 ); + assert( z[0]!=0 ); + if( z[1]==0 ){ + /* Wildcard of the form "?". Assign the next variable number */ + assert( z[0]=='?' ); + pExpr->iColumn = (ynVar)(++pParse->nVar); + }else{ + ynVar x = 0; + u32 n = sqlite3Strlen30(z); + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + pExpr->iColumn = x = (ynVar)i; + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + x = 0; + } + if( i>pParse->nVar ){ + pParse->nVar = (int)i; } - return WRC_Prune; - } - case TK_AGG_FUNCTION: { - /* The pNC->nDepth==0 test causes aggregate functions in subqueries - ** to be ignored */ - if( pNC->nDepth==0 ){ - /* Check to see if pExpr is a duplicate of another aggregate - ** function that is already in the pAggInfo structure - */ - struct AggInfo_func *pItem = pAggInfo->aFunc; - for(i=0; inFunc; i++, pItem++){ - if( sqlite3ExprCompare(pItem->pExpr, pExpr)==0 ){ - break; - } - } - if( i>=pAggInfo->nFunc ){ - /* pExpr is original. Make a new entry in pAggInfo->aFunc[] - */ - u8 enc = ENC(pParse->db); - i = addAggInfoFunc(pParse->db, pAggInfo); - if( i>=0 ){ - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pItem = &pAggInfo->aFunc[i]; - pItem->pExpr = pExpr; - pItem->iMem = ++pParse->nMem; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), - pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); - if( pExpr->flags & EP_Distinct ){ - pItem->iDistinct = pParse->nTab++; - }else{ - pItem->iDistinct = -1; - } - } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + ynVar i; + for(i=0; inzVar; i++){ + if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + pExpr->iColumn = x = (ynVar)i+1; + break; } - /* Make pExpr point to the appropriate pAggInfo->aFunc[] entry - */ - assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); - ExprSetIrreducible(pExpr); - pExpr->iAgg = (i16)i; - pExpr->pAggInfo = pAggInfo; - return WRC_Prune; } + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); } + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; + } + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); + } + } + } + if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "too many SQL variables"); } - return WRC_Continue; -} -static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ - NameContext *pNC = pWalker->u.pNC; - if( pNC->nDepth==0 ){ - pNC->nDepth++; - sqlite3WalkSelect(pWalker, pSelect); - pNC->nDepth--; - return WRC_Prune; - }else{ - return WRC_Continue; - } -} - -/* -** Analyze the given expression looking for aggregate functions and -** for variables that need to be added to the pParse->aAgg[] array. -** Make additional entries to the pParse->aAgg[] array as necessary. -** -** This routine should only be called after the expression has been -** analyzed by sqlite3ResolveExprNames(). -*/ -SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ - Walker w; - w.xExprCallback = analyzeAggregate; - w.xSelectCallback = analyzeAggregatesInSelect; - w.u.pNC = pNC; - assert( pNC->pSrcList!=0 ); - sqlite3WalkExpr(&w, pExpr); } /* -** Call sqlite3ExprAnalyzeAggregates() for every expression in an -** expression list. Return the number of errors. -** -** If an error is found, the analysis is cut short. +** Recursively delete an expression tree. */ -SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ - struct ExprList_item *pItem; - int i; - if( pList ){ - for(pItem=pList->a, i=0; inExpr; i++, pItem++){ - sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p==0 ) return; + /* Sanity check: Assert that the IntValue is non-negative if it exists */ + assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); + if( !ExprHasAnyProperty(p, EP_TokenOnly) ){ + sqlite3ExprDelete(db, p->pLeft); + sqlite3ExprDelete(db, p->pRight); + if( !ExprHasProperty(p, EP_Reduced) && (p->flags2 & EP2_MallocedToken)!=0 ){ + sqlite3DbFree(db, p->u.zToken); + } + if( ExprHasProperty(p, EP_xIsSelect) ){ + sqlite3SelectDelete(db, p->x.pSelect); + }else{ + sqlite3ExprListDelete(db, p->x.pList); } } + if( !ExprHasProperty(p, EP_Static) ){ + sqlite3DbFree(db, p); + } } /* -** Allocate a single new register for use to hold some intermediate result. +** Return the number of bytes allocated for the expression structure +** passed as the first argument. This is always one of EXPR_FULLSIZE, +** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. */ -SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ - if( pParse->nTempReg==0 ){ - return ++pParse->nMem; - } - return pParse->aTempReg[--pParse->nTempReg]; +static int exprStructSize(Expr *p){ + if( ExprHasProperty(p, EP_TokenOnly) ) return EXPR_TOKENONLYSIZE; + if( ExprHasProperty(p, EP_Reduced) ) return EXPR_REDUCEDSIZE; + return EXPR_FULLSIZE; } /* -** Deallocate a register, making available for reuse for some other -** purpose. +** The dupedExpr*Size() routines each return the number of bytes required +** to store a copy of an expression or expression tree. They differ in +** how much of the tree is measured. ** -** If a register is currently being used by the column cache, then -** the dallocation is deferred until the column cache line that uses -** the register becomes stale. +** dupedExprStructSize() Size of only the Expr structure +** dupedExprNodeSize() Size of Expr + space for token +** dupedExprSize() Expr + token + subtree components +** +*************************************************************************** +** +** The dupedExprStructSize() function returns two values OR-ed together: +** (1) the space required for a copy of the Expr structure only and +** (2) the EP_xxx flags that indicate what the structure size should be. +** The return values is always one of: +** +** EXPR_FULLSIZE +** EXPR_REDUCEDSIZE | EP_Reduced +** EXPR_TOKENONLYSIZE | EP_TokenOnly +** +** The size of the structure can be found by masking the return value +** of this routine with 0xfff. The flags can be found by masking the +** return value with EP_Reduced|EP_TokenOnly. +** +** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size +** (unreduced) Expr objects as they or originally constructed by the parser. +** During expression analysis, extra information is computed and moved into +** later parts of teh Expr object and that extra information might get chopped +** off if the expression is reduced. Note also that it does not work to +** make a EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** to reduce a pristine expression tree from the parser. The implementation +** of dupedExprStructSize() contain multiple assert() statements that attempt +** to enforce this constraint. */ -SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ - if( iReg && pParse->nTempRegaTempReg) ){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg==iReg ){ - p->tempReg = 1; - return; - } +static int dupedExprStructSize(Expr *p, int flags){ + int nSize; + assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ + if( 0==(flags&EXPRDUP_REDUCE) ){ + nSize = EXPR_FULLSIZE; + }else{ + assert( !ExprHasAnyProperty(p, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasProperty(p, EP_FromJoin) ); + assert( (p->flags2 & EP2_MallocedToken)==0 ); + assert( (p->flags2 & EP2_Irreducible)==0 ); + if( p->pLeft || p->pRight || p->pColl || p->x.pList ){ + nSize = EXPR_REDUCEDSIZE | EP_Reduced; + }else{ + nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; } - pParse->aTempReg[pParse->nTempReg++] = iReg; } + return nSize; } /* -** Allocate or deallocate a block of nReg consecutive registers +** This function returns the space in bytes required to store the copy +** of the Expr structure and a copy of the Expr.u.zToken string (if that +** string is defined.) */ -SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ - int i, n; - i = pParse->iRangeReg; - n = pParse->nRangeReg; - if( nReg<=n ){ - assert( !usedAsColumnCache(pParse, i, i+n-1) ); - pParse->iRangeReg += nReg; - pParse->nRangeReg -= nReg; - }else{ - i = pParse->nMem+1; - pParse->nMem += nReg; - } - return i; -} -SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ - sqlite3ExprCacheRemove(pParse, iReg, nReg); - if( nReg>pParse->nRangeReg ){ - pParse->nRangeReg = nReg; - pParse->iRangeReg = iReg; +static int dupedExprNodeSize(Expr *p, int flags){ + int nByte = dupedExprStructSize(p, flags) & 0xfff; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nByte += sqlite3Strlen30(p->u.zToken)+1; } + return ROUND8(nByte); } -/************** End of expr.c ************************************************/ -/************** Begin file alter.c *******************************************/ /* -** 2005 February 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** Return the number of bytes required to create a duplicate of the +** expression passed as the first argument. The second argument is a +** mask containing EXPRDUP_XXX flags. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** The value returned includes space to create a copy of the Expr struct +** itself and the buffer referred to by Expr.u.zToken, if any. ** -************************************************************************* -** This file contains C code routines that used to generate VDBE code -** that implements the ALTER TABLE command. +** If the EXPRDUP_REDUCE flag is set, then the return value includes +** space to duplicate all Expr nodes in the tree formed by Expr.pLeft +** and Expr.pRight variables (but not for any structures pointed to or +** descended from the Expr.x.pList or Expr.x.pSelect variables). */ +static int dupedExprSize(Expr *p, int flags){ + int nByte = 0; + if( p ){ + nByte = dupedExprNodeSize(p, flags); + if( flags&EXPRDUP_REDUCE ){ + nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); + } + } + return nByte; +} /* -** The code in this file only exists if we are not omitting the -** ALTER TABLE logic from the build. +** This function is similar to sqlite3ExprDup(), except that if pzBuffer +** is not NULL then *pzBuffer is assumed to point to a buffer large enough +** to store the copy of expression p, the copies of p->u.zToken +** (if applicable), and the copies of the p->pLeft and p->pRight expressions, +** if any. Before returning, *pzBuffer is set to the first byte passed the +** portion of the buffer copied into by this function. */ -#ifndef SQLITE_OMIT_ALTERTABLE - +static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ + Expr *pNew = 0; /* Value to return */ + if( p ){ + const int isReduced = (flags&EXPRDUP_REDUCE); + u8 *zAlloc; + u32 staticFlag = 0; -/* -** This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TABLE or -** CREATE INDEX command. The second is a table name. The table name in -** the CREATE TABLE or CREATE INDEX statement is replaced with the third -** argument and the result returned. Examples: -** -** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def') -** -> 'CREATE TABLE def(a, b, c)' -** -** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def') -** -> 'CREATE INDEX i ON def(a, b, c)' -*/ -static void renameTableFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); + assert( pzBuffer==0 || isReduced ); - int token; - Token tname; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + }else{ + zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); + } + pNew = (Expr *)zAlloc; - sqlite3 *db = sqlite3_context_db_handle(context); + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, flags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } + if( isReduced ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + int nSize = exprStructSize(p); + memcpy(zAlloc, p, nSize); + memset(&zAlloc[nSize], 0, EXPR_FULLSIZE-nSize); + } - UNUSED_PARAMETER(NotUsed); + /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; - /* The principle used to locate the table name in the CREATE TABLE - ** statement is that the table name is the first non-space token that - ** is immediately followed by a TK_LP or TK_USING token. - */ - if( zSql ){ - do { - if( !*zCsr ){ - /* Ran out of input before finding an opening bracket. Return NULL. */ - return; + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); } - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; + if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); + }else{ + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); + } + } - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - } while( token==TK_SPACE ); - assert( len>0 ); - } while( token!=TK_LP && token!=TK_USING ); + /* Fill in pNew->pLeft and pNew->pRight. */ + if( ExprHasAnyProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + zAlloc += dupedExprNodeSize(p, flags); + if( ExprHasProperty(pNew, EP_Reduced) ){ + pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); + pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); + } + if( pzBuffer ){ + *pzBuffer = zAlloc; + } + }else{ + pNew->flags2 = 0; + if( !ExprHasAnyProperty(p, EP_TokenOnly) ){ + pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); + } + } - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, - zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); + } } + return pNew; } /* -** This C function implements an SQL user function that is used by SQL code -** generated by the ALTER TABLE ... RENAME command to modify the definition -** of any foreign key constraints that use the table being renamed as the -** parent table. It is passed three arguments: +** The following group of routines make deep copies of expressions, +** expression lists, ID lists, and select statements. The copies can +** be deleted (by being passed to their respective ...Delete() routines) +** without effecting the originals. ** -** 1) The complete text of the CREATE TABLE statement being modified, -** 2) The old name of the table being renamed, and -** 3) The new name of the table being renamed. +** The expression list, ID, and source lists return by sqlite3ExprListDup(), +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** by subsequent calls to sqlite*ListAppend() routines. ** -** It returns the new CREATE TABLE statement. For example: +** Any tables that the SrcList might point to are not duplicated. ** -** sqlite_rename_parent('CREATE TABLE t1(a REFERENCES t2)', 't2', 't3') -** -> 'CREATE TABLE t1(a REFERENCES t3)' +** The flags parameter contains a combination of the EXPRDUP_XXX flags. +** If the EXPRDUP_REDUCE flag is set, then the structure returned is a +** truncated version of the usual Expr structure that will be stored as +** part of the in-memory representation of the database schema. */ -#ifndef SQLITE_OMIT_FOREIGN_KEY -static void renameParentFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - char *zOutput = 0; - char *zResult; - unsigned char const *zInput = sqlite3_value_text(argv[0]); - unsigned char const *zOld = sqlite3_value_text(argv[1]); - unsigned char const *zNew = sqlite3_value_text(argv[2]); - - unsigned const char *z; /* Pointer to token */ - int n; /* Length of token z */ - int token; /* Type of token */ - - UNUSED_PARAMETER(NotUsed); - for(z=zInput; *z; z=z+n){ - n = sqlite3GetToken(z, &token); - if( token==TK_REFERENCES ){ - char *zParent; - do { - z += n; - n = sqlite3GetToken(z, &token); - }while( token==TK_SPACE ); - - zParent = sqlite3DbStrNDup(db, (const char *)z, n); - if( zParent==0 ) break; - sqlite3Dequote(zParent); - if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ - char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", - (zOutput?zOutput:""), z-zInput, zInput, (const char *)zNew - ); - sqlite3DbFree(db, zOutput); - zOutput = zOut; - zInput = &z[n]; - } - sqlite3DbFree(db, zParent); - } - } - - zResult = sqlite3MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite3_result_text(context, zResult, -1, SQLITE_DYNAMIC); - sqlite3DbFree(db, zOutput); +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ + return exprDup(db, p, flags, 0); +} +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ + ExprList *pNew; + struct ExprList_item *pItem, *pOldItem; + int i; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->iECursor = 0; + pNew->nExpr = pNew->nAlloc = p->nExpr; + pNew->a = pItem = sqlite3DbMallocRaw(db, p->nExpr*sizeof(p->a[0]) ); + if( pItem==0 ){ + sqlite3DbFree(db, pNew); + return 0; + } + pOldItem = p->a; + for(i=0; inExpr; i++, pItem++, pOldItem++){ + Expr *pOldExpr = pOldItem->pExpr; + pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); + pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); + pItem->sortOrder = pOldItem->sortOrder; + pItem->done = 0; + pItem->iCol = pOldItem->iCol; + pItem->iAlias = pOldItem->iAlias; + } + return pNew; } -#endif - -#ifndef SQLITE_OMIT_TRIGGER -/* This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER -** statement. The second is a table name. The table name in the CREATE -** TRIGGER statement is replaced with the third argument and the result -** returned. This is analagous to renameTableFunc() above, except for CREATE -** TRIGGER, not CREATE INDEX and CREATE TABLE. -*/ -static void renameTriggerFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - int dist = 3; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - sqlite3 *db = sqlite3_context_db_handle(context); - UNUSED_PARAMETER(NotUsed); +/* +** If cursors, triggers, views and subqueries are all omitted from +** the build, then none of the following routines, except for +** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes +** called with a NULL argument. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ + || !defined(SQLITE_OMIT_SUBQUERY) +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ + SrcList *pNew; + int i; + int nByte; + if( p==0 ) return 0; + nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); + pNew = sqlite3DbMallocRaw(db, nByte ); + if( pNew==0 ) return 0; + pNew->nSrc = pNew->nAlloc = p->nSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pNewItem = &pNew->a[i]; + struct SrcList_item *pOldItem = &p->a[i]; + Table *pTab; + pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); + pNewItem->jointype = pOldItem->jointype; + pNewItem->iCursor = pOldItem->iCursor; + pNewItem->isPopulated = pOldItem->isPopulated; + pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); + pNewItem->notIndexed = pOldItem->notIndexed; + pNewItem->pIndex = pOldItem->pIndex; + pTab = pNewItem->pTab = pOldItem->pTab; + if( pTab ){ + pTab->nRef++; + } + pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); + pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); + pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); + pNewItem->colUsed = pOldItem->colUsed; + } + return pNew; +} +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ + IdList *pNew; + int i; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->nId = pNew->nAlloc = p->nId; + pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); + if( pNew->a==0 ){ + sqlite3DbFree(db, pNew); + return 0; + } + for(i=0; inId; i++){ + struct IdList_item *pNewItem = &pNew->a[i]; + struct IdList_item *pOldItem = &p->a[i]; + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->idx = pOldItem->idx; + } + return pNew; +} +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ + Select *pNew; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + if( pNew==0 ) return 0; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pPrior = sqlite3SelectDup(db, p->pPrior, flags); + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->pRightmost = 0; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->addrOpenEphm[2] = -1; + return pNew; +} +#else +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ + assert( p==0 ); + return 0; +} +#endif - /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediatedly - ** preceded by either TK_ON or TK_DOT and immediatedly followed by one - ** of TK_WHEN, TK_BEGIN or TK_FOR. - */ - if( zSql ){ - do { - if( !*zCsr ){ - /* Ran out of input before finding the table name. Return NULL. */ - return; - } +/* +** Add a new element to the end of an expression list. If pList is +** initially NULL, then create a new expression list. +** +** If a memory allocation error occurs, the entire list is freed and +** NULL is returned. If non-NULL is returned, then it is guaranteed +** that the new entry was successfully appended. +*/ +SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); + if( pList==0 ){ + goto no_mem; + } + assert( pList->nAlloc==0 ); + } + if( pList->nAlloc<=pList->nExpr ){ + struct ExprList_item *a; + int n = pList->nAlloc*2 + 4; + a = sqlite3DbRealloc(db, pList->a, n*sizeof(pList->a[0])); + if( a==0 ){ + goto no_mem; + } + pList->a = a; + pList->nAlloc = sqlite3DbMallocSize(db, a)/sizeof(a[0]); + } + assert( pList->a!=0 ); + if( 1 ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr++]; + memset(pItem, 0, sizeof(*pItem)); + pItem->pExpr = pExpr; + } + return pList; - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; +no_mem: + /* Avoid leaking memory if malloc has failed. */ + sqlite3ExprDelete(db, pExpr); + sqlite3ExprListDelete(db, pList); + return 0; +} - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - }while( token==TK_SPACE ); - assert( len>0 ); +/* +** Set the ExprList.a[].zName element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pName should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetName( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + Token *pName, /* Name to be added */ + int dequote /* True to cause the name to be dequoted */ +){ + assert( pList!=0 || pParse->db->mallocFailed!=0 ); + if( pList ){ + struct ExprList_item *pItem; + assert( pList->nExpr>0 ); + pItem = &pList->a[pList->nExpr-1]; + assert( pItem->zName==0 ); + pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); + if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); + } +} - /* Variable 'dist' stores the number of tokens read since the most - ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN - ** token is read and 'dist' equals 2, the condition stated above - ** to be met. - ** - ** Note that ON cannot be a database, table or column name, so - ** there is no need to worry about syntax like - ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc. - */ - dist++; - if( token==TK_DOT || token==TK_ON ){ - dist = 0; - } - } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) ); +/* +** Set the ExprList.a[].zSpan element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pSpan should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSpan( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + ExprSpan *pSpan /* The span to be added */ +){ + sqlite3 *db = pParse->db; + assert( pList!=0 || db->mallocFailed!=0 ); + if( pList ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; + assert( pList->nExpr>0 ); + assert( db->mallocFailed || pItem->pExpr==pSpan->pExpr ); + sqlite3DbFree(db, pItem->zSpan); + pItem->zSpan = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); + } +} - /* Variable tname now contains the token that is the old table-name - ** in the CREATE TRIGGER statement. - */ - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, - zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); +/* +** If the expression list pEList contains more than iLimit elements, +** leave an error message in pParse. +*/ +SQLITE_PRIVATE void sqlite3ExprListCheckLength( + Parse *pParse, + ExprList *pEList, + const char *zObject +){ + int mx = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + testcase( pEList && pEList->nExpr==mx ); + testcase( pEList && pEList->nExpr==mx+1 ); + if( pEList && pEList->nExpr>mx ){ + sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); } } -#endif /* !SQLITE_OMIT_TRIGGER */ /* -** Register built-in functions used to help implement ALTER TABLE +** Delete an entire expression list. */ -SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static SQLITE_WSD FuncDef aAlterTableFuncs[] = { - FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), -#ifndef SQLITE_OMIT_TRIGGER - FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), -#endif -#ifndef SQLITE_OMIT_FOREIGN_KEY - FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), -#endif - }; +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); - - for(i=0; ia!=0 || (pList->nExpr==0 && pList->nAlloc==0) ); + assert( pList->nExpr<=pList->nAlloc ); + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprDelete(db, pItem->pExpr); + sqlite3DbFree(db, pItem->zName); + sqlite3DbFree(db, pItem->zSpan); } + sqlite3DbFree(db, pList->a); + sqlite3DbFree(db, pList); } /* -** This function is used to create the text of expressions of the form: +** These routines are Walker callbacks. Walker.u.pi is a pointer +** to an integer. These routines are checking an expression to see +** if it is a constant. Set *Walker.u.pi to 0 if the expression is +** not constant. ** -** name= OR name= OR ... +** These callback routines are used to implement the following: ** -** If argument zWhere is NULL, then a pointer string containing the text -** "name=" is returned, where is the quoted version -** of the string passed as argument zConstant. The returned buffer is -** allocated using sqlite3DbMalloc(). It is the responsibility of the -** caller to ensure that it is eventually freed. +** sqlite3ExprIsConstant() +** sqlite3ExprIsConstantNotJoin() +** sqlite3ExprIsConstantOrFunction() ** -** If argument zWhere is not NULL, then the string returned is -** " OR name=", where is the contents of zWhere. -** In this case zWhere is passed to sqlite3DbFree() before returning. -** */ -static char *whereOrName(sqlite3 *db, char *zWhere, char *zConstant){ - char *zNew; - if( !zWhere ){ - zNew = sqlite3MPrintf(db, "name=%Q", zConstant); - }else{ - zNew = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, zConstant); - sqlite3DbFree(db, zWhere); +static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + + /* If pWalker->u.i is 3 then any term of the expression that comes from + ** the ON or USING clauses of a join disqualifies the expression + ** from being considered constant. */ + if( pWalker->u.i==3 && ExprHasAnyProperty(pExpr, EP_FromJoin) ){ + pWalker->u.i = 0; + return WRC_Abort; } - return zNew; + + switch( pExpr->op ){ + /* Consider functions to be constant if all their arguments are constant + ** and pWalker->u.i==2 */ + case TK_FUNCTION: + if( pWalker->u.i==2 ) return 0; + /* Fall through */ + case TK_ID: + case TK_COLUMN: + case TK_AGG_FUNCTION: + case TK_AGG_COLUMN: + testcase( pExpr->op==TK_ID ); + testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + testcase( pExpr->op==TK_AGG_COLUMN ); + pWalker->u.i = 0; + return WRC_Abort; + default: + testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ + testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ + return WRC_Continue; + } +} +static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->u.i = 0; + return WRC_Abort; +} +static int exprIsConst(Expr *p, int initFlag){ + Walker w; + w.u.i = initFlag; + w.xExprCallback = exprNodeIsConstant; + w.xSelectCallback = selectNodeIsConstant; + sqlite3WalkExpr(&w, p); + return w.u.i; } -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) /* -** Generate the text of a WHERE expression which can be used to select all -** tables that have foreign key constraints that refer to table pTab (i.e. -** constraints for which pTab is the parent table) from the sqlite_master -** table. +** Walk an expression tree. Return 1 if the expression is constant +** and 0 if it involves variables or function calls. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. */ -static char *whereForeignKeys(Parse *pParse, Table *pTab){ - FKey *p; - char *zWhere = 0; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - zWhere = whereOrName(pParse->db, zWhere, p->pFrom->zName); - } - return zWhere; +SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ + return exprIsConst(p, 1); } -#endif /* -** Generate the text of a WHERE expression which can be used to select all -** temporary triggers on table pTab from the sqlite_temp_master table. If -** table pTab has no temporary triggers, or is itself stored in the -** temporary database, NULL is returned. +** Walk an expression tree. Return 1 if the expression is constant +** that does no originate from the ON or USING clauses of a join. +** Return 0 if it involves variables or function calls or terms from +** an ON or USING clause. */ -static char *whereTempTriggers(Parse *pParse, Table *pTab){ - Trigger *pTrig; - char *zWhere = 0; - const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */ +SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ + return exprIsConst(p, 3); +} - /* If the table is not located in the temp-db (in which case NULL is - ** returned, loop through the tables list of triggers. For each trigger - ** that is not part of the temp-db schema, add a clause to the WHERE - ** expression being built up in zWhere. - */ - if( pTab->pSchema!=pTempSchema ){ - sqlite3 *db = pParse->db; - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - if( pTrig->pSchema==pTempSchema ){ - zWhere = whereOrName(db, zWhere, pTrig->zName); - } - } - } - if( zWhere ){ - char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); - sqlite3DbFree(pParse->db, zWhere); - zWhere = zNew; - } - return zWhere; +/* +** Walk an expression tree. Return 1 if the expression is constant +** or a function call with constant arguments. Return and 0 if there +** are any variables. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p){ + return exprIsConst(p, 2); } /* -** Generate code to drop and reload the internal representation of table -** pTab from the database, including triggers and temporary triggers. -** Argument zName is the name of the table in the database schema at -** the time the generated code is executed. This can be different from -** pTab->zName if this function is being called to code part of an -** "ALTER TABLE RENAME TO" statement. +** If the expression p codes a constant integer that is small enough +** to fit in a 32-bit integer, return 1 and put the value of the integer +** in *pValue. If the expression is not an integer or if it is too big +** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. */ -static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ - Vdbe *v; - char *zWhere; - int iDb; /* Index of database containing pTab */ -#ifndef SQLITE_OMIT_TRIGGER - Trigger *pTrig; -#endif +SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ + int rc = 0; - v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - assert( iDb>=0 ); + /* If an expression is an integer literal that fits in a signed 32-bit + ** integer, then the EP_IntValue flag will have already been set */ + assert( p->op!=TK_INTEGER || (p->flags & EP_IntValue)!=0 + || sqlite3GetInt32(p->u.zToken, &rc)==0 ); -#ifndef SQLITE_OMIT_TRIGGER - /* Drop any table triggers from the internal schema. */ - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); - assert( iTrigDb==iDb || iTrigDb==1 ); - sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->zName, 0); + if( p->flags & EP_IntValue ){ + *pValue = p->u.iValue; + return 1; } -#endif - - /* Drop the table and index from the internal schema. */ - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - - /* Reload the table, index and permanent trigger schemas. */ - zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); - if( !zWhere ) return; - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); - -#ifndef SQLITE_OMIT_TRIGGER - /* Now, if the table is not stored in the temp database, reload any temp - ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); + switch( p->op ){ + case TK_UPLUS: { + rc = sqlite3ExprIsInteger(p->pLeft, pValue); + break; + } + case TK_UMINUS: { + int v; + if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + *pValue = -v; + rc = 1; + } + break; + } + default: break; } -#endif + return rc; } /* -** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" -** command. -*/ -SQLITE_PRIVATE void sqlite3AlterRenameTable( - Parse *pParse, /* Parser context. */ - SrcList *pSrc, /* The table to rename. */ - Token *pName /* The new table name. */ -){ - int iDb; /* Database that contains the table */ - char *zDb; /* Name of database iDb */ - Table *pTab; /* Table being renamed */ - char *zName = 0; /* NULL-terminated version of pName */ - sqlite3 *db = pParse->db; /* Database connection */ - int nTabName; /* Number of UTF-8 characters in zTabName */ - const char *zTabName; /* Original name of the table */ - Vdbe *v; -#ifndef SQLITE_OMIT_TRIGGER - char *zWhere = 0; /* Where clause to locate temp triggers */ -#endif - VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ - int savedDbFlags; /* Saved value of db->flags */ - - savedDbFlags = db->flags; - if( NEVER(db->mallocFailed) ) goto exit_rename_table; - assert( pSrc->nSrc==1 ); - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); - if( !pTab ) goto exit_rename_table; - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - zDb = db->aDb[iDb].zName; - db->flags |= SQLITE_PreferBuiltin; - - /* Get a NULL terminated version of the new table name. */ - zName = sqlite3NameFromToken(db, pName); - if( !zName ) goto exit_rename_table; - - /* Check that a table or index named 'zName' does not already exist - ** in database iDb. If so, this is an error. - */ - if( sqlite3FindTable(db, zName, zDb) || sqlite3FindIndex(db, zName, zDb) ){ - sqlite3ErrorMsg(pParse, - "there is already another table or index with this name: %s", zName); - goto exit_rename_table; +** Return FALSE if there is no chance that the expression can be NULL. +** +** If the expression might be NULL or if the expression is too complex +** to tell return TRUE. +** +** This routine is used as an optimization, to skip OP_IsNull opcodes +** when we know that a value cannot be NULL. Hence, a false positive +** (returning TRUE when in fact the expression can never be NULL) might +** be a small performance hit but is otherwise harmless. On the other +** hand, a false negative (returning FALSE when the result could be NULL) +** will likely result in an incorrect answer. So when in doubt, return +** TRUE. +*/ +SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ + u8 op; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + return 0; + default: + return 1; } +} - /* Make sure it is not a system table being altered, or a reserved name - ** that the table is being renamed to. - */ - if( sqlite3Strlen30(pTab->zName)>6 - && 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) - ){ - sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName); - goto exit_rename_table; - } - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ - goto exit_rename_table; +/* +** Generate an OP_IsNull instruction that tests register iReg and jumps +** to location iDest if the value in iReg is NULL. The value in iReg +** was computed by pExpr. If we can look at pExpr at compile-time and +** determine that it can never generate a NULL, then the OP_IsNull operation +** can be omitted. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump( + Vdbe *v, /* The VDBE under construction */ + const Expr *pExpr, /* Only generate OP_IsNull if this expr can be NULL */ + int iReg, /* Test the value in this register for NULL */ + int iDest /* Jump here if the value is null */ +){ + if( sqlite3ExprCanBeNull(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest); } +} -#ifndef SQLITE_OMIT_VIEW - if( pTab->pSelect ){ - sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); - goto exit_rename_table; +/* +** Return TRUE if the given expression is a constant which would be +** unchanged by OP_Affinity with the affinity given in the second +** argument. +** +** This routine is used to determine if the OP_Affinity operation +** can be omitted. When in doubt return FALSE. A false negative +** is harmless. A false positive, however, can result in the wrong +** answer. +*/ +SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ + u8 op; + if( aff==SQLITE_AFF_NONE ) return 1; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: { + return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; + } + case TK_FLOAT: { + return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; + } + case TK_STRING: { + return aff==SQLITE_AFF_TEXT; + } + case TK_BLOB: { + return 1; + } + case TK_COLUMN: { + assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ + return p->iColumn<0 + && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); + } + default: { + return 0; + } } -#endif +} -#ifndef SQLITE_OMIT_AUTHORIZATION - /* Invoke the authorization callback. */ - if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ - goto exit_rename_table; - } -#endif +/* +** Return TRUE if the given string is a row-id column name. +*/ +SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ + if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1; + if( sqlite3StrICmp(z, "ROWID")==0 ) return 1; + if( sqlite3StrICmp(z, "OID")==0 ) return 1; + return 0; +} -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3ViewGetColumnNames(pParse, pTab) ){ - goto exit_rename_table; - } - if( IsVirtual(pTab) ){ - pVTab = sqlite3GetVTable(db, pTab); - if( pVTab->pVtab->pModule->xRename==0 ){ - pVTab = 0; - } +/* +** Return true if we are able to the IN operator optimization on a +** query of the form +** +** x IN (SELECT ...) +** +** Where the SELECT... clause is as specified by the parameter to this +** routine. +** +** The Select object passed in has already been preprocessed and no +** errors have been found. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +static int isCandidateForInOpt(Select *p){ + SrcList *pSrc; + ExprList *pEList; + Table *pTab; + if( p==0 ) return 0; /* right-hand side of IN is SELECT */ + if( p->pPrior ) return 0; /* Not a compound SELECT */ + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + return 0; /* No DISTINCT keyword and no aggregate functions */ } -#endif + assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ + if( p->pLimit ) return 0; /* Has no LIMIT clause */ + assert( p->pOffset==0 ); /* No LIMIT means no OFFSET */ + if( p->pWhere ) return 0; /* Has no WHERE clause */ + pSrc = p->pSrc; + assert( pSrc!=0 ); + if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ + if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ + pTab = pSrc->a[0].pTab; + if( NEVER(pTab==0) ) return 0; + assert( pTab->pSelect==0 ); /* FROM clause is not a view */ + if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ + pEList = p->pEList; + if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ + if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ + return 1; +} +#endif /* SQLITE_OMIT_SUBQUERY */ - /* Begin a transaction and code the VerifyCookie for database iDb. - ** Then modify the schema cookie (since the ALTER TABLE modifies the - ** schema). Open a statement transaction if the table is a virtual - ** table. - */ - v = sqlite3GetVdbe(pParse); - if( v==0 ){ - goto exit_rename_table; - } - sqlite3BeginWriteOperation(pParse, pVTab!=0, iDb); - sqlite3ChangeCookie(pParse, iDb); +/* +** This function is used by the implementation of the IN (...) operator. +** It's job is to find or create a b-tree structure that may be used +** either to test for membership of the (...) set or to iterate through +** its members, skipping duplicates. +** +** The index of the cursor opened on the b-tree (database table, database index +** or ephermal table) is stored in pX->iTable before this function returns. +** The returned value of this function indicates the b-tree type, as follows: +** +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX - The cursor was opened on a database index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. +** +** An existing b-tree may only be used if the SELECT is of the simple +** form: +** +** SELECT FROM
    +** +** If the prNotFound parameter is 0, then the b-tree will be used to iterate +** through the set members, skipping any duplicates. In this case an +** epheremal table must be used unless the selected is guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or it +** has a UNIQUE constraint or UNIQUE index. +** +** If the prNotFound parameter is not 0, then the b-tree will be used +** for fast set membership tests. In this case an epheremal table must +** be used unless is an INTEGER PRIMARY KEY or an index can +** be found with as its left-most column. +** +** When the b-tree is being used for membership tests, the calling function +** needs to know whether or not the structure contains an SQL NULL +** value in order to correctly evaluate expressions like "X IN (Y, Z)". +** If there is any chance that the (...) might contain a NULL value at +** runtime, then a register is allocated and the register number written +** to *prNotFound. If there is no chance that the (...) contains a +** NULL value, then *prNotFound is left unchanged. +** +** If a register is allocated and its location stored in *prNotFound, then +** its initial value is NULL. If the (...) does not remain constant +** for the duration of the query (i.e. the SELECT within the (...) +** is a correlated subquery) then the value of the allocated register is +** reset to NULL each time the subquery is rerun. This allows the +** caller to use vdbe code equivalent to the following: +** +** if( register==NULL ){ +** has_null = +** register = 1 +** } +** +** in order to avoid running the +** test more often than is necessary. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ + Select *p; /* SELECT to the right of IN operator */ + int eType = 0; /* Type of RHS table. IN_INDEX_* */ + int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ - /* If this is a virtual table, invoke the xRename() function if - ** one is defined. The xRename() callback will modify the names - ** of any resources used by the v-table implementation (including other - ** SQLite tables) that are identified by the name of the virtual table. + assert( pX->op==TK_IN ); + + /* Check to see if an existing table or index can be used to + ** satisfy the query. This is preferable to generating a new + ** ephemeral table. */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pVTab ){ - int i = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); - sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); - sqlite3MayAbort(pParse); - } -#endif + p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); + if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ + sqlite3 *db = pParse->db; /* Database connection */ + Expr *pExpr = p->pEList->a[0].pExpr; /* Expression */ + int iCol = pExpr->iColumn; /* Index of column */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + Table *pTab = p->pSrc->a[0].pTab; /* Table
    . */ + int iDb; /* Database idx for pTab */ + + /* Code an OP_VerifyCookie and OP_TableLock for
    . */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* figure out how many UTF-8 characters are in zName */ - zTabName = pTab->zName; - nTabName = sqlite3Utf8CharLen(zTabName, -1); + /* This function is only called from two places. In both cases the vdbe + ** has already been allocated. So assume sqlite3GetVdbe() is always + ** successful here. + */ + assert(v); + if( iCol<0 ){ + int iMem = ++pParse->nMem; + int iAddr; -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - /* If foreign-key support is enabled, rewrite the CREATE TABLE - ** statements corresponding to all child tables of foreign key constraints - ** for which the renamed table is the parent table. */ - if( (zWhere=whereForeignKeys(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".%s SET " - "sql = sqlite_rename_parent(sql, %Q, %Q) " - "WHERE %s;", zDb, SCHEMA_TABLE(iDb), zTabName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } - } -#endif + iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); - /* Modify the sqlite_master table to use the new table name. */ - sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET " -#ifdef SQLITE_OMIT_TRIGGER - "sql = sqlite_rename_table(sql, %Q), " -#else - "sql = CASE " - "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)" - "ELSE sqlite_rename_table(sql, %Q) END, " -#endif - "tbl_name = %Q, " - "name = CASE " - "WHEN type='table' THEN %Q " - "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " - "'sqlite_autoindex_' || %Q || substr(name,%d+18) " - "ELSE name END " - "WHERE tbl_name=%Q AND " - "(type='table' OR type='index' OR type='trigger');", - zDb, SCHEMA_TABLE(iDb), zName, zName, zName, -#ifndef SQLITE_OMIT_TRIGGER - zName, -#endif - zName, nTabName, zTabName - ); + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + eType = IN_INDEX_ROWID; -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* If the sqlite_sequence table exists in this database, then update - ** it with the new table name. - */ - if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", - zDb, zName, pTab->zName); - } -#endif + sqlite3VdbeJumpHere(v, iAddr); + }else{ + Index *pIdx; /* Iterator variable */ -#ifndef SQLITE_OMIT_TRIGGER - /* If there are TEMP triggers on this table, modify the sqlite_temp_master - ** table. Don't do this if the table being ALTERed is itself located in - ** the temp database. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE sqlite_temp_master SET " - "sql = sqlite_rename_trigger(sql, %Q), " - "tbl_name = %Q " - "WHERE %s;", zName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } -#endif + /* The collation sequence used by the comparison. If an index is to + ** be used in place of a temp-table, it must be ordered according + ** to this collation sequence. */ + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - FKey *p; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - Table *pFrom = p->pFrom; - if( pFrom!=pTab ){ - reloadTableSchema(pParse, p->pFrom, pFrom->zName); + /* Check that the affinity that will be used to perform the + ** comparison is the same as the affinity of the column. If + ** it is not, it is not possible to use any index. + */ + char aff = comparisonAffinity(pX); + int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); + + for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ + if( (pIdx->aiColumn[0]==iCol) + && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq + && (!mustBeUnique || (pIdx->nColumn==1 && pIdx->onError!=OE_None)) + ){ + int iMem = ++pParse->nMem; + int iAddr; + char *pKey; + + pKey = (char *)sqlite3IndexKeyinfo(pParse, pIdx); + iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); + + sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, + pKey,P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); + eType = IN_INDEX_INDEX; + + sqlite3VdbeJumpHere(v, iAddr); + if( prNotFound && !pTab->aCol[iCol].notNull ){ + *prNotFound = ++pParse->nMem; + } + } } } } -#endif - - /* Drop and reload the internal table schema. */ - reloadTableSchema(pParse, pTab, zName); - -exit_rename_table: - sqlite3SrcListDelete(db, pSrc); - sqlite3DbFree(db, zName); - db->flags = savedDbFlags; -} - -/* -** Generate code to make sure the file format number is at least minFormat. -** The generated code will increase the file format number if necessary. -*/ -SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ - Vdbe *v; - v = sqlite3GetVdbe(pParse); - /* The VDBE should have been allocated before this routine is called. - ** If that allocation failed, we would have quit before reaching this - ** point */ - if( ALWAYS(v) ){ - int r1 = sqlite3GetTempReg(pParse); - int r2 = sqlite3GetTempReg(pParse); - int j1; - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); - j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); - sqlite3VdbeJumpHere(v, j1); - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempReg(pParse, r2); + if( eType==0 ){ + /* Could not found an existing table or index to use as the RHS b-tree. + ** We will have to generate an ephemeral table to do the job. + */ + double savedNQueryLoop = pParse->nQueryLoop; + int rMayHaveNull = 0; + eType = IN_INDEX_EPH; + if( prNotFound ){ + *prNotFound = rMayHaveNull = ++pParse->nMem; + }else{ + testcase( pParse->nQueryLoop>(double)1 ); + pParse->nQueryLoop = (double)1; + if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ + eType = IN_INDEX_ROWID; + } + } + sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + pParse->nQueryLoop = savedNQueryLoop; + }else{ + pX->iTable = iTab; } + return eType; } +#endif /* -** This function is called after an "ALTER TABLE ... ADD" statement -** has been parsed. Argument pColDef contains the text of the new -** column definition. +** Generate code for scalar subqueries used as a subquery expression, EXISTS, +** or IN operators. Examples: ** -** The Table structure pParse->pNewTable was extended to include -** the new column during parsing. +** (SELECT a FROM b) -- subquery +** EXISTS (SELECT a FROM b) -- EXISTS subquery +** x IN (4,5,11) -- IN operator with list on right-hand side +** x IN (SELECT a FROM b) -- IN operator with subquery on the right +** +** The pExpr parameter describes the expression that contains the IN +** operator or subquery. +** +** If parameter isRowid is non-zero, then expression pExpr is guaranteed +** to be of the form " IN (?, ?, ?)", where is a reference +** to some integer key column of a table B-Tree. In this case, use an +** intkey B-Tree to store the set of IN(...) values instead of the usual +** (slower) variable length keys B-Tree. +** +** If rMayHaveNull is non-zero, that means that the operation is an IN +** (not a SELECT or EXISTS) and that the RHS might contains NULLs. +** Furthermore, the IN is in a WHERE clause and that we really want +** to iterate over the RHS of the IN operator in order to quickly locate +** all corresponding LHS elements. All this routine does is initialize +** the register given by rMayHaveNull to NULL. Calling routines will take +** care of changing this register value to non-NULL if the RHS is NULL-free. +** +** If rMayHaveNull is zero, that means that the subquery is being used +** for membership testing only. There is no need to initialize any +** registers to indicate the presense or absence of NULLs on the RHS. +** +** For a SELECT or EXISTS operator, return the register that holds the +** result. For IN operators or if an error occurs, the return value is 0. */ -SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ - Table *pNew; /* Copy of pParse->pNewTable */ - Table *pTab; /* Table being altered */ - int iDb; /* Database number */ - const char *zDb; /* Database name */ - const char *zTab; /* Table name */ - char *zCol; /* Null-terminated column definition */ - Column *pCol; /* The new column */ - Expr *pDflt; /* Default value for the new column */ - sqlite3 *db; /* The database connection; */ - - db = pParse->db; - if( pParse->nErr || db->mallocFailed ) return; - pNew = pParse->pNewTable; - assert( pNew ); +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3CodeSubselect( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ + int rMayHaveNull, /* Register that records whether NULLs exist in RHS */ + int isRowid /* If true, LHS of IN operator is a rowid */ +){ + int testAddr = 0; /* One-time test address */ + int rReg = 0; /* Register storing resulting */ + Vdbe *v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return 0; + sqlite3ExprCachePush(pParse); - assert( sqlite3BtreeHoldsAllMutexes(db) ); - iDb = sqlite3SchemaToIndex(db, pNew->pSchema); - zDb = db->aDb[iDb].zName; - zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ - pCol = &pNew->aCol[pNew->nCol-1]; - pDflt = pCol->pDflt; - pTab = sqlite3FindTable(db, zTab, zDb); - assert( pTab ); + /* This code must be run in its entirety every time it is encountered + ** if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can run this code just once + ** save the results, and reuse the same result on subsequent invocations. + */ + if( !ExprHasAnyProperty(pExpr, EP_VarSelect) && !pParse->pTriggerTab ){ + int mem = ++pParse->nMem; + sqlite3VdbeAddOp1(v, OP_If, mem); + testAddr = sqlite3VdbeAddOp2(v, OP_Integer, 1, mem); + assert( testAddr>0 || pParse->db->mallocFailed ); + } -#ifndef SQLITE_OMIT_AUTHORIZATION - /* Invoke the authorization callback. */ - if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ - return; +#ifndef SQLITE_OMIT_EXPLAIN + if( pParse->explain==2 ){ + char *zMsg = sqlite3MPrintf( + pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); } #endif - /* If the default value for the new column was specified with a - ** literal NULL, then set pDflt to 0. This simplifies checking - ** for an SQL NULL default below. - */ - if( pDflt && pDflt->op==TK_NULL ){ - pDflt = 0; - } + switch( pExpr->op ){ + case TK_IN: { + char affinity; /* Affinity of the LHS of the IN */ + KeyInfo keyInfo; /* Keyinfo for the generated table */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ - /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. - ** If there is a NOT NULL constraint, then the default value for the - ** column must not be NULL. - */ - if( pCol->isPrimKey ){ - sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); - return; - } - if( pNew->pIndex ){ - sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column"); - return; - } - if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a REFERENCES column with non-NULL default value"); - return; - } - if( pCol->notNull && !pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a NOT NULL column with default value NULL"); - return; - } + if( rMayHaveNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); + } - /* Ensure the default expression is something that sqlite3ValueFromExpr() - ** can handle (i.e. not CURRENT_TIME etc.) - */ - if( pDflt ){ - sqlite3_value *pVal; - if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ - db->mallocFailed = 1; - return; - } - if( !pVal ){ - sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); - return; + affinity = sqlite3ExprAffinity(pLeft); + + /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' + ** expression it is handled the same way. An ephemeral table is + ** filled with single-field index keys representing the results + ** from the SELECT or the . + ** + ** If the 'x' expression is a column value, or the SELECT... + ** statement returns a column value, then the affinity of that + ** column is used to build the index keys. If both 'x' and the + ** SELECT... statement are columns, then numeric affinity is used + ** if either column has NUMERIC or INTEGER affinity. If neither + ** 'x' nor the SELECT... statement are columns, then numeric affinity + ** is used. + */ + pExpr->iTable = pParse->nTab++; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); + if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + memset(&keyInfo, 0, sizeof(keyInfo)); + keyInfo.nField = 1; + + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* Case 1: expr IN (SELECT ...) + ** + ** Generate code to write the results of the select into the temporary + ** table allocated and opened above. + */ + SelectDest dest; + ExprList *pEList; + + assert( !isRowid ); + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.affinity = (u8)affinity; + assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pExpr->x.pSelect->iLimit = 0; + if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ + return 0; + } + pEList = pExpr->x.pSelect->pEList; + if( ALWAYS(pEList!=0 && pEList->nExpr>0) ){ + keyInfo.aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, + pEList->a[0].pExpr); + } + }else if( ALWAYS(pExpr->x.pList!=0) ){ + /* Case 2: expr IN (exprlist) + ** + ** For each expression, build an index key from the evaluation and + ** store it in the temporary table. If is a column, then use + ** that columns affinity when building index keys. If is not + ** a column, use numeric affinity. + */ + int i; + ExprList *pList = pExpr->x.pList; + struct ExprList_item *pItem; + int r1, r2, r3; + + if( !affinity ){ + affinity = SQLITE_AFF_NONE; + } + keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + + /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, r2); + for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ + Expr *pE2 = pItem->pExpr; + int iValToIns; + + /* If the expression is not constant then we will need to + ** disable the test that was generated above that makes sure + ** this code only executes once. Because for a non-constant + ** expression we need to rerun this code each time. + */ + if( testAddr && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, testAddr-1, 2); + testAddr = 0; + } + + /* Evaluate the expression and insert it into the temp table */ + if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){ + sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns); + }else{ + r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); + if( isRowid ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, + sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, r3, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pExpr->iTable, r2); + } + } + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( !isRowid ){ + sqlite3VdbeChangeP4(v, addr, (void *)&keyInfo, P4_KEYINFO); + } + break; } - sqlite3ValueFree(pVal); - } - /* Modify the CREATE TABLE statement. */ - zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); - if( zCol ){ - char *zEnd = &zCol[pColDef->n-1]; - int savedDbFlags = db->flags; - while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ - *zEnd-- = '\0'; + case TK_EXISTS: + case TK_SELECT: + default: { + /* If this has to be a scalar SELECT. Generate code to put the + ** value of this select in a memory cell and record the number + ** of the memory cell in iColumn. If this is an EXISTS, write + ** an integer 0 (not exists) or 1 (exists) into a memory cell + ** and record that memory cell in iColumn. + */ + Select *pSel; /* SELECT statement to encode */ + SelectDest dest; /* How to deal with SELECt result */ + + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); + assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; + sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); + if( pExpr->op==TK_SELECT ){ + dest.eDest = SRT_Mem; + sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iParm); + VdbeComment((v, "Init subquery result")); + }else{ + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iParm); + VdbeComment((v, "Init EXISTS result")); + } + sqlite3ExprDelete(pParse->db, pSel->pLimit); + pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, + &sqlite3IntTokens[1]); + pSel->iLimit = 0; + if( sqlite3Select(pParse, pSel, &dest) ){ + return 0; + } + rReg = dest.iParm; + ExprSetIrreducible(pExpr); + break; } - db->flags |= SQLITE_PreferBuiltin; - sqlite3NestedParse(pParse, - "UPDATE \"%w\".%s SET " - "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " - "WHERE type = 'table' AND name = %Q", - zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1, - zTab - ); - sqlite3DbFree(db, zCol); - db->flags = savedDbFlags; } - /* If the default value of the new column is NULL, then set the file - ** format to 2. If the default value of the new column is not NULL, - ** the file format becomes 3. - */ - sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2); + if( testAddr ){ + sqlite3VdbeJumpHere(v, testAddr-1); + } + sqlite3ExprCachePop(pParse, 1); - /* Reload the schema of the modified table. */ - reloadTableSchema(pParse, pTab, pTab->zName); + return rReg; } +#endif /* SQLITE_OMIT_SUBQUERY */ +#ifndef SQLITE_OMIT_SUBQUERY /* -** This function is called by the parser after the table-name in -** an "ALTER TABLE ADD" statement is parsed. Argument -** pSrc is the full-name of the table being altered. +** Generate code for an IN expression. ** -** This routine makes a (partial) copy of the Table structure -** for the table being altered and sets Parse.pNewTable to point -** to it. Routines called by the parser as the column definition -** is parsed (i.e. sqlite3AddColumn()) add the new Column data to -** the copy. The copy of the Table structure is deleted by tokenize.c -** after parsing is finished. +** x IN (SELECT ...) +** x IN (value, value, ...) ** -** Routine sqlite3AlterFinishAddColumn() will be called to complete -** coding the "ALTER TABLE ... ADD" statement. +** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) +** is an array of zero or more values. The expression is true if the LHS is +** contained within the RHS. The value of the expression is unknown (NULL) +** if the LHS is NULL or if the LHS is not contained within the RHS and the +** RHS contains one or more NULL values. +** +** This routine generates code will jump to destIfFalse if the LHS is not +** contained within the RHS. If due to NULLs we cannot determine if the LHS +** is contained in the RHS then jump to destIfNull. If the LHS is contained +** within the RHS then fall through. */ -SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ - Table *pNew; - Table *pTab; - Vdbe *v; - int iDb; - int i; - int nAlloc; - sqlite3 *db = pParse->db; - - /* Look up the table being altered. */ - assert( pParse->pNewTable==0 ); - assert( sqlite3BtreeHoldsAllMutexes(db) ); - if( db->mallocFailed ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); - if( !pTab ) goto exit_begin_add_column; +static void sqlite3ExprCodeIN( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The IN expression */ + int destIfFalse, /* Jump here if LHS is not contained in the RHS */ + int destIfNull /* Jump here if the results are unknown due to NULLs */ +){ + int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ + char affinity; /* Comparison affinity to use */ + int eType; /* Type of the RHS */ + int r1; /* Temporary use register */ + Vdbe *v; /* Statement under construction */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - sqlite3ErrorMsg(pParse, "virtual tables may not be altered"); - goto exit_begin_add_column; - } -#endif + /* Compute the RHS. After this step, the table with cursor + ** pExpr->iTable will contains the values that make up the RHS. + */ + v = pParse->pVdbe; + assert( v!=0 ); /* OOM detected prior to this routine */ + VdbeNoopComment((v, "begin IN expr")); + eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull); - /* Make sure this is not an attempt to ALTER a view. */ - if( pTab->pSelect ){ - sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); - goto exit_begin_add_column; - } + /* Figure out the affinity to use to create a key from the results + ** of the expression. affinityStr stores a static string suitable for + ** P4 of OP_MakeRecord. + */ + affinity = comparisonAffinity(pExpr); - assert( pTab->addColOffset>0 ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + /* Code the LHS, the from " IN (...)". + */ + sqlite3ExprCachePush(pParse); + r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCode(pParse, pExpr->pLeft, r1); - /* Put a copy of the Table struct in Parse.pNewTable for the - ** sqlite3AddColumn() function and friends to modify. But modify - ** the name by adding an "sqlite_altertab_" prefix. By adding this - ** prefix, we insure that the name will not collide with an existing - ** table because user table are not allowed to have the "sqlite_" - ** prefix on their name. + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. */ - pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); - if( !pNew ) goto exit_begin_add_column; - pParse->pNewTable = pNew; - pNew->nRef = 1; - pNew->nCol = pTab->nCol; - assert( pNew->nCol>0 ); - nAlloc = (((pNew->nCol-1)/8)*8)+8; - assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); - pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); - pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); - if( !pNew->aCol || !pNew->zName ){ - db->mallocFailed = 1; - goto exit_begin_add_column; - } - memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); - for(i=0; inCol; i++){ - Column *pCol = &pNew->aCol[i]; - pCol->zName = sqlite3DbStrDup(db, pCol->zName); - pCol->zColl = 0; - pCol->zType = 0; - pCol->pDflt = 0; - pCol->zDflt = 0; + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); } - pNew->pSchema = db->aDb[iDb].pSchema; - pNew->addColOffset = pTab->addColOffset; - pNew->nRef = 1; - /* Begin a transaction and increment the schema cookie. */ - sqlite3BeginWriteOperation(pParse, 0, iDb); - v = sqlite3GetVdbe(pParse); - if( !v ) goto exit_begin_add_column; - sqlite3ChangeCookie(pParse, iDb); + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree + */ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + }else{ + /* In this case, the RHS is an index b-tree. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); -exit_begin_add_column: - sqlite3SrcListDelete(db, pSrc); - return; + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. + */ + if( rRhsHasNull==0 || destIfFalse==destIfNull ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1, j2, j3; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the presence of NULLs in the RHS does not matter, so jump + ** over all of the code that follows. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + + /* Here we begin generating code that runs if the LHS is not + ** contained within the RHS. Generate additional code that + ** tests the RHS for NULLs. If the RHS contains a NULL then + ** jump to destIfNull. If there are no NULLs in the RHS then + ** jump to destIfFalse. + */ + j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull); + j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); + sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull); + sqlite3VdbeJumpHere(v, j3); + sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1); + sqlite3VdbeJumpHere(v, j2); + + /* Jump to the appropriate target depending on whether or not + ** the RHS contains a NULL + */ + sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + + /* The OP_Found at the top of this branch jumps here when true, + ** causing the overall IN expression evaluation to fall through. + */ + sqlite3VdbeJumpHere(v, j1); + } + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ExprCachePop(pParse, 1); + VdbeComment((v, "end IN expr")); } -#endif /* SQLITE_ALTER_TABLE */ +#endif /* SQLITE_OMIT_SUBQUERY */ -/************** End of alter.c ***********************************************/ -/************** Begin file analyze.c *****************************************/ /* -** 2005 July 8 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code associated with the ANALYZE command. +** Duplicate an 8-byte value */ -#ifndef SQLITE_OMIT_ANALYZE +static char *dup8bytes(Vdbe *v, const char *in){ + char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); + if( out ){ + memcpy(out, in, 8); + } + return out; +} +#ifndef SQLITE_OMIT_FLOATING_POINT /* -** This routine generates code that opens the sqlite_stat1 table for -** writing with cursor iStatCur. If the library was built with the -** SQLITE_ENABLE_STAT2 macro defined, then the sqlite_stat2 table is -** opened for writing using cursor (iStatCur+1) +** Generate an instruction that will put the floating point +** value described by z[0..n-1] into register iMem. ** -** If the sqlite_stat1 tables does not previously exist, it is created. -** Similarly, if the sqlite_stat2 table does not exist and the library -** is compiled with SQLITE_ENABLE_STAT2 defined, it is created. -** -** Argument zWhere may be a pointer to a buffer containing a table name, -** or it may be a NULL pointer. If it is not NULL, then all entries in -** the sqlite_stat1 and (if applicable) sqlite_stat2 tables associated -** with the named table are deleted. If zWhere==0, then code is generated -** to delete all stat table entries. +** The z[] string will probably not be zero-terminated. But the +** z[n] character is guaranteed to be something that does not look +** like the continuation of the number. */ -static void openStatTable( - Parse *pParse, /* Parsing context */ - int iDb, /* The database we are looking in */ - int iStatCur, /* Open the sqlite_stat1 table on this cursor */ - const char *zWhere /* Delete entries associated with this table */ -){ - static const struct { - const char *zName; - const char *zCols; - } aTable[] = { - { "sqlite_stat1", "tbl,idx,stat" }, -#ifdef SQLITE_ENABLE_STAT2 - { "sqlite_stat2", "tbl,idx,sampleno,sample" }, +static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ + if( ALWAYS(z!=0) ){ + double value; + char *zV; + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ + if( negateFlag ) value = -value; + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); + } +} #endif - }; - int aRoot[] = {0, 0}; - u8 aCreateTbl[] = {0, 0}; - - int i; - sqlite3 *db = pParse->db; - Db *pDb; - Vdbe *v = sqlite3GetVdbe(pParse); - if( v==0 ) return; - assert( sqlite3BtreeHoldsAllMutexes(db) ); - assert( sqlite3VdbeDb(v)==db ); - pDb = &db->aDb[iDb]; - for(i=0; izName))==0 ){ - /* The sqlite_stat[12] table does not exist. Create it. Note that a - ** side-effect of the CREATE TABLE statement is to leave the rootpage - ** of the new table in register pParse->regRoot. This is important - ** because the OpenWrite opcode below will be needing it. */ - sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols - ); - aRoot[i] = pParse->regRoot; - aCreateTbl[i] = 1; +/* +** Generate an instruction that will put the integer describe by +** text z[0..n-1] into register iMem. +** +** Expr.u.zToken is always UTF8 and zero-terminated. +*/ +static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ + Vdbe *v = pParse->pVdbe; + if( pExpr->flags & EP_IntValue ){ + int i = pExpr->u.iValue; + assert( i>=0 ); + if( negFlag ) i = -i; + sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); + }else{ + int c; + i64 value; + const char *z = pExpr->u.zToken; + assert( z!=0 ); + c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + if( c==0 || (c==2 && negFlag) ){ + char *zV; + if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); }else{ - /* The table already exists. If zWhere is not NULL, delete all entries - ** associated with the table zWhere. If zWhere is NULL, delete the - ** entire contents of the table. */ - aRoot[i] = pStat->tnum; - sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); - if( zWhere ){ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE tbl=%Q", pDb->zName, zTab, zWhere - ); - }else{ - /* The sqlite_stat[12] table already exists. Delete all rows. */ - sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); - } +#ifdef SQLITE_OMIT_FLOATING_POINT + sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); +#else + codeReal(v, z, negFlag, iMem); +#endif } } - - /* Open the sqlite_stat[12] tables for writing. */ - for(i=0; idb; /* Database handle */ - Index *pIdx; /* An index to being analyzed */ - int iIdxCur; /* Cursor open on index being analyzed */ - Vdbe *v; /* The virtual machine being built up */ - int i; /* Loop counter */ - int topOfLoop; /* The top of the loop */ - int endOfLoop; /* The end of the loop */ - int addr = 0; /* The address of an instruction */ - int jZeroRows = 0; /* Jump from here if number of rows is zero */ - int iDb; /* Index of database containing pTab */ - int regTabname = iMem++; /* Register containing table name */ - int regIdxname = iMem++; /* Register containing index name */ - int regSampleno = iMem++; /* Register containing next sample number */ - int regCol = iMem++; /* Content of a column analyzed table */ - int regRec = iMem++; /* Register holding completed record */ - int regTemp = iMem++; /* Temporary use register */ - int regRowid = iMem++; /* Rowid for the inserted record */ - -#ifdef SQLITE_ENABLE_STAT2 - int regTemp2 = iMem++; /* Temporary use register */ - int regSamplerecno = iMem++; /* Index of next sample to record */ - int regRecno = iMem++; /* Current sample index */ - int regLast = iMem++; /* Index of last sample to record */ - int regFirst = iMem++; /* Index of first sample to record */ -#endif - - v = sqlite3GetVdbe(pParse); - if( v==0 || NEVER(pTab==0) ){ - return; - } - if( pTab->tnum==0 ){ - /* Do not gather statistics on views or virtual tables */ - return; - } - if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ - /* Do not gather statistics on system tables */ - return; - } - assert( sqlite3BtreeHoldsAllMutexes(db) ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( iDb>=0 ); -#ifndef SQLITE_OMIT_AUTHORIZATION - if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, - db->aDb[iDb].zName ) ){ - return; - } -#endif - - /* Establish a read-lock on the table at the shared-cache level. */ - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - - iIdxCur = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int nCol = pIdx->nColumn; - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); - - if( iMem+1+(nCol*2)>pParse->nMem ){ - pParse->nMem = iMem+1+(nCol*2); +static void cacheEntryClear(Parse *pParse, struct yColCache *p){ + if( p->tempReg ){ + if( pParse->nTempRegaTempReg) ){ + pParse->aTempReg[pParse->nTempReg++] = p->iReg; } + p->tempReg = 0; + } +} - /* Open a cursor to the index to be analyzed. */ - assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); - sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, - (char *)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIdx->zName)); - /* Populate the register containing the index name. */ - sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); +/* +** Record in the column cache that a particular column from a +** particular table is stored in a particular register. +*/ +SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ + int i; + int minLru; + int idxLru; + struct yColCache *p; -#ifdef SQLITE_ENABLE_STAT2 + assert( iReg>0 ); /* Register numbers are always positive */ + assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ - /* If this iteration of the loop is generating code to analyze the - ** first index in the pTab->pIndex list, then register regLast has - ** not been populated. In this case populate it now. */ - if( pTab->pIndex==pIdx ){ - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regSamplerecno); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2-1, regTemp); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2, regTemp2); + /* The SQLITE_ColumnCache flag disables the column cache. This is used + ** for testing only - to verify that SQLite always gets the same answer + ** with and without the column cache. + */ + if( pParse->db->flags & SQLITE_ColumnCache ) return; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regLast); - sqlite3VdbeAddOp2(v, OP_Null, 0, regFirst); - addr = sqlite3VdbeAddOp3(v, OP_Lt, regSamplerecno, 0, regLast); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regFirst); - sqlite3VdbeAddOp3(v, OP_Multiply, regLast, regTemp, regLast); - sqlite3VdbeAddOp2(v, OP_AddImm, regLast, SQLITE_INDEX_SAMPLES*2-2); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regLast); - sqlite3VdbeJumpHere(v, addr); + /* First replace any existing entry. + ** + ** Actually, the way the column cache is currently used, we are guaranteed + ** that the object will never already be in cache. Verify this guarantee. + */ +#ifndef NDEBUG + for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ + cacheEntryClear(pParse, p); + p->iLevel = pParse->iCacheLevel; + p->iReg = iReg; + p->lru = pParse->iCacheCnt++; + return; } - - /* Zero the regSampleno and regRecno registers. */ - sqlite3VdbeAddOp2(v, OP_Integer, 0, regSampleno); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRecno); - sqlite3VdbeAddOp2(v, OP_Copy, regFirst, regSamplerecno); #endif - - /* The block of memory cells initialized here is used as follows. - ** - ** iMem: - ** The total number of rows in the table. - ** - ** iMem+1 .. iMem+nCol: - ** Number of distinct entries in index considering the - ** left-most N columns only, where N is between 1 and nCol, - ** inclusive. - ** - ** iMem+nCol+1 .. Mem+2*nCol: - ** Previous value of indexed columns, from left to right. - ** - ** Cells iMem through iMem+nCol are initialized to 0. The others are - ** initialized to contain an SQL NULL. - */ - for(i=0; i<=nCol; i++){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem+i); - } - for(i=0; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); + } #endif - sqlite3VdbeAddOp3(v, OP_Ne, regCol, 0, iMem+nCol+i+1); - /**** TODO: add collating sequence *****/ - sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); - } - if( db->mallocFailed ){ - /* If a malloc failure has occurred, then the result of the expression - ** passed as the second argument to the call to sqlite3VdbeJumpHere() - ** below may be negative. Which causes an assert() to fail (or an - ** out-of-bounds write if SQLITE_DEBUG is not defined). */ + /* Find an empty slot and replace it */ + for(i=0, p=pParse->aColCache; iiReg==0 ){ + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; return; } - sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); - for(i=0; i0 then it is always the case the D>0 so division by zero - ** is never possible. - */ - sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); - if( jZeroRows==0 ){ - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); - } - for(i=0; ipIndex==0 ){ - sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); - VdbeComment((v, "%s", pTab->zName)); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); - sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); - }else{ - assert( jZeroRows>0 ); - addr = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, jZeroRows); + /* Replace the last recently used */ + minLru = 0x7fffffff; + idxLru = -1; + for(i=0, p=pParse->aColCache; ilrulru; + } } - sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - if( pParse->nMemnMem = regRec; - if( jZeroRows ){ - sqlite3VdbeJumpHere(v, addr); + if( ALWAYS(idxLru>=0) ){ + p = &pParse->aColCache[idxLru]; + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; + return; } } /* -** Generate code that will cause the most recent index analysis to -** be loaded into internal hash tables where is can be used. +** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. +** Purge the range of registers from the column cache. */ -static void loadAnalysis(Parse *pParse, int iDb){ - Vdbe *v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); +SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ + int i; + int iLast = iReg + nReg - 1; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg; + if( r>=iReg && r<=iLast ){ + cacheEntryClear(pParse, p); + p->iReg = 0; + } } } /* -** Generate code that will do an analysis of an entire database +** Remember the current column cache context. Any new entries added +** added to the column cache after this call are removed when the +** corresponding pop occurs. */ -static void analyzeDatabase(Parse *pParse, int iDb){ - sqlite3 *db = pParse->db; - Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ - HashElem *k; - int iStatCur; - int iMem; +SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ + pParse->iCacheLevel++; +} - sqlite3BeginWriteOperation(pParse, 0, iDb); - iStatCur = pParse->nTab; - pParse->nTab += 2; - openStatTable(pParse, iDb, iStatCur, 0); - iMem = pParse->nMem+1; - for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ - Table *pTab = (Table*)sqliteHashData(k); - analyzeOneTable(pParse, pTab, iStatCur, iMem); +/* +** Remove from the column cache any entries that were added since the +** the previous N Push operations. In other words, restore the cache +** to the state it was in N Pushes ago. +*/ +SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse, int N){ + int i; + struct yColCache *p; + assert( N>0 ); + assert( pParse->iCacheLevel>=N ); + pParse->iCacheLevel -= N; + for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ + cacheEntryClear(pParse, p); + p->iReg = 0; + } } - loadAnalysis(pParse, iDb); } /* -** Generate code that will do an analysis of a single table in -** a database. +** When a cached column is reused, make sure that its register is +** no longer available as a temp register. ticket #3879: that same +** register might be in the cache in multiple places, so be sure to +** get them all. */ -static void analyzeTable(Parse *pParse, Table *pTab){ - int iDb; - int iStatCur; +static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ + int i; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg==iReg ){ + p->tempReg = 0; + } + } +} - assert( pTab!=0 ); - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - iStatCur = pParse->nTab; - pParse->nTab += 2; - openStatTable(pParse, iDb, iStatCur, pTab->zName); - analyzeOneTable(pParse, pTab, iStatCur, pParse->nMem+1); - loadAnalysis(pParse, iDb); +/* +** Generate code to extract the value of the iCol-th column of a table. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( + Vdbe *v, /* The VDBE under construction */ + Table *pTab, /* The table containing the value */ + int iTabCur, /* The cursor for this table */ + int iCol, /* Index of the column to extract */ + int regOut /* Extract the valud into this register */ +){ + if( iCol<0 || iCol==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); + }else{ + int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; + sqlite3VdbeAddOp3(v, op, iTabCur, iCol, regOut); + } + if( iCol>=0 ){ + sqlite3ColumnDefault(v, pTab, iCol, regOut); + } } /* -** Generate code for the ANALYZE command. The parser calls this routine -** when it recognizes an ANALYZE command. -** -** ANALYZE -- 1 -** ANALYZE -- 2 -** ANALYZE ?.? -- 3 +** Generate code that will extract the iColumn-th column from +** table pTab and store the column value in a register. An effort +** is made to store the column value in register iReg, but this is +** not guaranteed. The location of the column value is returned. ** -** Form 1 causes all indices in all attached databases to be analyzed. -** Form 2 analyzes all indices the single database named. -** Form 3 analyzes all indices associated with the named table. +** There must be an open cursor to pTab in iTable when this routine +** is called. If iColumn<0 then code is generated that extracts the rowid. */ -SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ - sqlite3 *db = pParse->db; - int iDb; +SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg /* Store results here */ +){ + Vdbe *v = pParse->pVdbe; int i; - char *z, *zDb; - Table *pTab; - Token *pTableName; - - /* Read the database schema. If an error occurs, leave an error message - ** and code in pParse and return NULL. */ - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - return; - } + struct yColCache *p; - assert( pName2!=0 || pName1==0 ); - if( pName1==0 ){ - /* Form 1: Analyze everything */ - for(i=0; inDb; i++){ - if( i==1 ) continue; /* Do not analyze the TEMP database */ - analyzeDatabase(pParse, i); - } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - pTab = sqlite3LocateTable(pParse, 0, z, 0); - sqlite3DbFree(db, z); - if( pTab ){ - analyzeTable(pParse, pTab); - } - } + for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ + p->lru = pParse->iCacheCnt++; + sqlite3ExprCachePinRegister(pParse, p->iReg); + return p->iReg; } - }else{ - /* Form 3: Analyze the fully qualified table name */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); - if( iDb>=0 ){ - zDb = db->aDb[iDb].zName; - z = sqlite3NameFromToken(db, pTableName); - if( z ){ - pTab = sqlite3LocateTable(pParse, 0, z, zDb); - sqlite3DbFree(db, z); - if( pTab ){ - analyzeTable(pParse, pTab); - } - } - } - } + } + assert( v!=0 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + return iReg; } /* -** Used to pass information from the analyzer reader through to the -** callback routine. -*/ -typedef struct analysisInfo analysisInfo; -struct analysisInfo { - sqlite3 *db; - const char *zDatabase; -}; - -/* -** This callback is invoked once for each index when reading the -** sqlite_stat1 table. -** -** argv[0] = name of the table -** argv[1] = name of the index (might be NULL) -** argv[2] = results of analysis - on integer for each column -** -** Entries for which argv[1]==NULL simply record the number of rows in -** the table. +** Clear all column cache entries. */ -static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ - analysisInfo *pInfo = (analysisInfo*)pData; - Index *pIndex; - Table *pTable; - int i, c, n; - unsigned int v; - const char *z; - - assert( argc==3 ); - UNUSED_PARAMETER2(NotUsed, argc); +SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ + int i; + struct yColCache *p; - if( argv==0 || argv[0]==0 || argv[2]==0 ){ - return 0; - } - pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); - if( pTable==0 ){ - return 0; - } - if( argv[1] ){ - pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); - }else{ - pIndex = 0; - } - n = pIndex ? pIndex->nColumn : 0; - z = argv[2]; - for(i=0; *z && i<=n; i++){ - v = 0; - while( (c=z[0])>='0' && c<='9' ){ - v = v*10 + c - '0'; - z++; + for(i=0, p=pParse->aColCache; iiReg ){ + cacheEntryClear(pParse, p); + p->iReg = 0; } - if( i==0 ) pTable->nRowEst = v; - if( pIndex==0 ) break; - pIndex->aiRowEst[i] = v; - if( *z==' ' ) z++; } - return 0; } /* -** If the Index.aSample variable is not NULL, delete the aSample[] array -** and its contents. +** Record the fact that an affinity change has occurred on iCount +** registers starting with iStart. */ -SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT2 - if( pIdx->aSample ){ - int j; - for(j=0; jaSample[j]; - if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ - sqlite3DbFree(db, p->u.z); - } - } - sqlite3DbFree(db, pIdx->aSample); - } -#else - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pIdx); -#endif +SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ + sqlite3ExprCacheRemove(pParse, iStart, iCount); } /* -** Load the content of the sqlite_stat1 and sqlite_stat2 tables. The -** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat2 are used to populate the -** Index.aSample[] arrays. -** -** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT2 was defined -** during compilation and the sqlite_stat2 table is present, no data is -** read from it. -** -** If SQLITE_ENABLE_STAT2 was defined during compilation and the -** sqlite_stat2 table is not present in the database, SQLITE_ERROR is -** returned. However, in this case, data is read from the sqlite_stat1 -** table (if it is present) before returning. -** -** If an OOM error occurs, this function always sets db->mallocFailed. -** This means if the caller does not care about other errors, the return -** code may be ignored. +** Generate code to move content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. */ -SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ - analysisInfo sInfo; - HashElem *i; - char *zSql; - int rc; - - assert( iDb>=0 && iDbnDb ); - assert( db->aDb[iDb].pBt!=0 ); - assert( sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - - /* Clear any prior statistics */ - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ - Index *pIdx = sqliteHashData(i); - sqlite3DefaultRowEst(pIdx); - sqlite3DeleteIndexSamples(db, pIdx); - pIdx->aSample = 0; - } - - /* Check to make sure the sqlite_stat1 table exists */ - sInfo.db = db; - sInfo.zDatabase = db->aDb[iDb].zName; - if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ - return SQLITE_ERROR; - } - - /* Load new statistics out of the sqlite_stat1 table */ - zSql = sqlite3MPrintf(db, - "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); - sqlite3DbFree(db, zSql); - } - - - /* Load the statistics from the sqlite_stat2 table. */ -#ifdef SQLITE_ENABLE_STAT2 - if( rc==SQLITE_OK && !sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase) ){ - rc = SQLITE_ERROR; - } - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - - zSql = sqlite3MPrintf(db, - "SELECT idx,sampleno,sample FROM %Q.sqlite_stat2", sInfo.zDatabase); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - sqlite3DbFree(db, zSql); - } - - if( rc==SQLITE_OK ){ - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - char *zIndex = (char *)sqlite3_column_text(pStmt, 0); - Index *pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase); - if( pIdx ){ - int iSample = sqlite3_column_int(pStmt, 1); - if( iSample=0 ){ - int eType = sqlite3_column_type(pStmt, 2); - - if( pIdx->aSample==0 ){ - static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; - pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); - if( pIdx->aSample==0 ){ - db->mallocFailed = 1; - break; - } - memset(pIdx->aSample, 0, sz); - } - - assert( pIdx->aSample ); - { - IndexSample *pSample = &pIdx->aSample[iSample]; - pSample->eType = (u8)eType; - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - pSample->u.r = sqlite3_column_double(pStmt, 2); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const char *z = (const char *)( - (eType==SQLITE_BLOB) ? - sqlite3_column_blob(pStmt, 2): - sqlite3_column_text(pStmt, 2) - ); - int n = sqlite3_column_bytes(pStmt, 2); - if( n>24 ){ - n = 24; - } - pSample->nByte = (u8)n; - if( n < 1){ - pSample->u.z = 0; - }else{ - pSample->u.z = sqlite3DbStrNDup(0, z, n); - if( pSample->u.z==0 ){ - db->mallocFailed = 1; - break; - } - } - } - } - } - } - } - rc = sqlite3_finalize(pStmt); +SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ + int i; + struct yColCache *p; + if( NEVER(iFrom==iTo) ) return; + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); + for(i=0, p=pParse->aColCache; iiReg; + if( x>=iFrom && xiReg += iTo-iFrom; } } -#endif - - if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; - } - return rc; } - -#endif /* SQLITE_OMIT_ANALYZE */ - -/************** End of analyze.c *********************************************/ -/************** Begin file attach.c ******************************************/ /* -** 2003 April 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code used to implement the ATTACH and DETACH commands. +** Generate code to copy content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. */ +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, int iFrom, int iTo, int nReg){ + int i; + if( NEVER(iFrom==iTo) ) return; + for(i=0; ipVdbe, OP_Copy, iFrom+i, iTo+i); + } +} -#ifndef SQLITE_OMIT_ATTACH +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* -** Resolve an expression that was part of an ATTACH or DETACH statement. This -** is slightly different from resolving a normal SQL expression, because simple -** identifiers are treated as strings, not possible column names or aliases. -** -** i.e. if the parser sees: -** -** ATTACH DATABASE abc AS def -** -** it treats the two expressions as literal strings 'abc' and 'def' instead of -** looking for columns of the same name. -** -** This only applies to the root node of pExpr, so the statement: -** -** ATTACH DATABASE abc||def AS 'db2' +** Return true if any register in the range iFrom..iTo (inclusive) +** is used as part of the column cache. ** -** will fail because neither abc or def can be resolved. +** This routine is used within assert() and testcase() macros only +** and does not appear in a normal build. */ -static int resolveAttachExpr(NameContext *pName, Expr *pExpr) -{ - int rc = SQLITE_OK; - if( pExpr ){ - if( pExpr->op!=TK_ID ){ - rc = sqlite3ResolveExprNames(pName, pExpr); - if( rc==SQLITE_OK && !sqlite3ExprIsConstant(pExpr) ){ - sqlite3ErrorMsg(pName->pParse, "invalid name: \"%s\"", pExpr->u.zToken); - return SQLITE_ERROR; - } - }else{ - pExpr->op = TK_STRING; - } +static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ + int i; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg; + if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ } - return rc; + return 0; } +#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ /* -** An SQL user-function registered to do the work of an ATTACH statement. The -** three arguments to the function come directly from an attach statement: -** -** ATTACH DATABASE x AS y KEY z -** -** SELECT sqlite_attach(x, y, z) +** Generate code into the current Vdbe to evaluate the given +** expression. Attempt to store the results in register "target". +** Return the register where results are stored. ** -** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the -** third argument. +** With this routine, there is no guarantee that results will +** be stored in target. The result might be stored in some other +** register if it is convenient to do so. The calling function +** must check the return code and move the results to the desired +** register. */ -static void attachFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - int i; - int rc = 0; - sqlite3 *db = sqlite3_context_db_handle(context); - const char *zName; - const char *zFile; - Db *aNew; - char *zErrDyn = 0; - - UNUSED_PARAMETER(NotUsed); - - zFile = (const char *)sqlite3_value_text(argv[0]); - zName = (const char *)sqlite3_value_text(argv[1]); - if( zFile==0 ) zFile = ""; - if( zName==0 ) zName = ""; +SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; /* The VM under construction */ + int op; /* The opcode being coded */ + int inReg = target; /* Results stored in register inReg */ + int regFree1 = 0; /* If non-zero free this temporary register */ + int regFree2 = 0; /* If non-zero free this temporary register */ + int r1, r2, r3, r4; /* Various register numbers */ + sqlite3 *db = pParse->db; /* The database connection */ - /* Check for the following errors: - ** - ** * Too many attached databases, - ** * Transaction currently open - ** * Specified database name already being used. - */ - if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ - zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", - db->aLimit[SQLITE_LIMIT_ATTACHED] - ); - goto attach_error; - } - if( !db->autoCommit ){ - zErrDyn = sqlite3MPrintf(db, "cannot ATTACH database within transaction"); - goto attach_error; - } - for(i=0; inDb; i++){ - char *z = db->aDb[i].zName; - assert( z && zName ); - if( sqlite3StrICmp(z, zName)==0 ){ - zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); - goto attach_error; - } + assert( target>0 && target<=pParse->nMem ); + if( v==0 ){ + assert( pParse->db->mallocFailed ); + return 0; } - /* Allocate the new entry in the db->aDb[] array and initialise the schema - ** hash tables. - */ - if( db->aDb==db->aDbStatic ){ - aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); - if( aNew==0 ) return; - memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); + if( pExpr==0 ){ + op = TK_NULL; }else{ - aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); - if( aNew==0 ) return; - } - db->aDb = aNew; - aNew = &db->aDb[db->nDb]; - memset(aNew, 0, sizeof(*aNew)); - - /* Open the database file. If the btree is successfully opened, use - ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. - */ - rc = sqlite3BtreeOpen(zFile, db, &aNew->pBt, 0, - db->openFlags | SQLITE_OPEN_MAIN_DB); - db->nDb++; - if( rc==SQLITE_CONSTRAINT ){ - rc = SQLITE_ERROR; - zErrDyn = sqlite3MPrintf(db, "database is already attached"); - }else if( rc==SQLITE_OK ){ - Pager *pPager; - aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); - if( !aNew->pSchema ){ - rc = SQLITE_NOMEM; - }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ - zErrDyn = sqlite3MPrintf(db, - "attached databases must use the same text encoding as main database"); - rc = SQLITE_ERROR; - } - pPager = sqlite3BtreePager(aNew->pBt); - sqlite3PagerLockingMode(pPager, db->dfltLockMode); - sqlite3BtreeSecureDelete(aNew->pBt, - sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); - } - aNew->safety_level = 3; - aNew->zName = sqlite3DbStrDup(db, zName); - if( rc==SQLITE_OK && aNew->zName==0 ){ - rc = SQLITE_NOMEM; + op = pExpr->op; } - - -#ifdef SQLITE_HAS_CODEC - if( rc==SQLITE_OK ){ - extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); - int nKey; - char *zKey; - int t = sqlite3_value_type(argv[2]); - switch( t ){ - case SQLITE_INTEGER: - case SQLITE_FLOAT: - zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); - rc = SQLITE_ERROR; - break; - - case SQLITE_TEXT: - case SQLITE_BLOB: - nKey = sqlite3_value_bytes(argv[2]); - zKey = (char *)sqlite3_value_blob(argv[2]); - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + switch( op ){ + case TK_AGG_COLUMN: { + AggInfo *pAggInfo = pExpr->pAggInfo; + struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; + if( !pAggInfo->directMode ){ + assert( pCol->iMem>0 ); + inReg = pCol->iMem; break; - - case SQLITE_NULL: - /* No key specified. Use the key from the main database */ - sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + }else if( pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, + pCol->iSorterColumn, target); break; + } + /* Otherwise, fall thru into the TK_COLUMN case */ } - } -#endif - - /* If the file was opened successfully, read the schema for the new database. - ** If this fails, or if opening the file failed, then close the file and - ** remove the entry from the db->aDb[] array. i.e. put everything back the way - ** we found it. - */ - if( rc==SQLITE_OK ){ - sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrDyn); - sqlite3BtreeLeaveAll(db); - } - if( rc ){ - int iDb = db->nDb - 1; - assert( iDb>=2 ); - if( db->aDb[iDb].pBt ){ - sqlite3BtreeClose(db->aDb[iDb].pBt); - db->aDb[iDb].pBt = 0; - db->aDb[iDb].pSchema = 0; + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + assert( pParse->ckBase>0 ); + inReg = pExpr->iColumn + pParse->ckBase; + }else{ + inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + pExpr->iColumn, pExpr->iTable, target); + } + break; } - sqlite3ResetInternalSchema(db, 0); - db->nDb = iDb; - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; - sqlite3DbFree(db, zErrDyn); - zErrDyn = sqlite3MPrintf(db, "out of memory"); - }else if( zErrDyn==0 ){ - zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + case TK_INTEGER: { + codeInteger(pParse, pExpr, 0, target); + break; } - goto attach_error; - } - - return; - -attach_error: - /* Return an error if we get here */ - if( zErrDyn ){ - sqlite3_result_error(context, zErrDyn, -1); - sqlite3DbFree(db, zErrDyn); - } - if( rc ) sqlite3_result_error_code(context, rc); -} - -/* -** An SQL user-function registered to do the work of an DETACH statement. The -** three arguments to the function come directly from a detach statement: -** -** DETACH DATABASE x -** -** SELECT sqlite_detach(x) -*/ -static void detachFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - const char *zName = (const char *)sqlite3_value_text(argv[0]); - sqlite3 *db = sqlite3_context_db_handle(context); - int i; - Db *pDb = 0; - char zErr[128]; - - UNUSED_PARAMETER(NotUsed); - - if( zName==0 ) zName = ""; - for(i=0; inDb; i++){ - pDb = &db->aDb[i]; - if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; - } - - if( i>=db->nDb ){ - sqlite3_snprintf(sizeof(zErr),zErr, "no such database: %s", zName); - goto detach_error; - } - if( i<2 ){ - sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); - goto detach_error; - } - if( !db->autoCommit ){ - sqlite3_snprintf(sizeof(zErr), zErr, - "cannot DETACH database within transaction"); - goto detach_error; - } - if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){ - sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); - goto detach_error; - } - - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - pDb->pSchema = 0; - sqlite3ResetInternalSchema(db, 0); - return; - -detach_error: - sqlite3_result_error(context, zErr, -1); -} - -/* -** This procedure generates VDBE code for a single invocation of either the -** sqlite_detach() or sqlite_attach() SQL user functions. -*/ -static void codeAttach( - Parse *pParse, /* The parser context */ - int type, /* Either SQLITE_ATTACH or SQLITE_DETACH */ - FuncDef const *pFunc,/* FuncDef wrapper for detachFunc() or attachFunc() */ - Expr *pAuthArg, /* Expression to pass to authorization callback */ - Expr *pFilename, /* Name of database file */ - Expr *pDbname, /* Name of the database to use internally */ - Expr *pKey /* Database key for encryption extension */ -){ - int rc; - NameContext sName; - Vdbe *v; - sqlite3* db = pParse->db; - int regArgs; - - memset(&sName, 0, sizeof(NameContext)); - sName.pParse = pParse; - - if( - SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) || - SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || - SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) - ){ - pParse->nErr++; - goto attach_end; - } - -#ifndef SQLITE_OMIT_AUTHORIZATION - if( pAuthArg ){ - char *zAuthArg = pAuthArg->u.zToken; - if( NEVER(zAuthArg==0) ){ - goto attach_end; +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pExpr->u.zToken, 0, target); + break; } - rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0); - if(rc!=SQLITE_OK ){ - goto attach_end; +#endif + case TK_STRING: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + break; } - } -#endif /* SQLITE_OMIT_AUTHORIZATION */ - - - v = sqlite3GetVdbe(pParse); - regArgs = sqlite3GetTempRange(pParse, 4); - sqlite3ExprCode(pParse, pFilename, regArgs); - sqlite3ExprCode(pParse, pDbname, regArgs+1); - sqlite3ExprCode(pParse, pKey, regArgs+2); - - assert( v || db->mallocFailed ); - if( v ){ - sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); - assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); - sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); - - /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this - ** statement only). For DETACH, set it to false (expire all existing - ** statements). - */ - sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); - } - -attach_end: - sqlite3ExprDelete(db, pFilename); - sqlite3ExprDelete(db, pDbname); - sqlite3ExprDelete(db, pKey); -} - -/* -** Called by the parser to compile a DETACH statement. -** -** DETACH pDbname -*/ -SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ - static const FuncDef detach_func = { - 1, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ - 0, /* pUserData */ - 0, /* pNext */ - detachFunc, /* xFunc */ - 0, /* xStep */ - 0, /* xFinalize */ - "sqlite_detach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ - }; - codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); -} - -/* -** Called by the parser to compile an ATTACH statement. -** -** ATTACH p AS pDbname KEY pKey -*/ -SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ - static const FuncDef attach_func = { - 3, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ - 0, /* pUserData */ - 0, /* pNext */ - attachFunc, /* xFunc */ - 0, /* xStep */ - 0, /* xFinalize */ - "sqlite_attach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ - }; - codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); -} -#endif /* SQLITE_OMIT_ATTACH */ - -/* -** Initialize a DbFixer structure. This routine must be called prior -** to passing the structure to one of the sqliteFixAAAA() routines below. -** -** The return value indicates whether or not fixation is required. TRUE -** means we do need to fix the database references, FALSE means we do not. -*/ -SQLITE_PRIVATE int sqlite3FixInit( - DbFixer *pFix, /* The fixer to be initialized */ - Parse *pParse, /* Error messages will be written here */ - int iDb, /* This is the database that must be used */ - const char *zType, /* "view", "trigger", or "index" */ - const Token *pName /* Name of the view, trigger, or index */ -){ - sqlite3 *db; - - if( NEVER(iDb<0) || iDb==1 ) return 0; - db = pParse->db; - assert( db->nDb>iDb ); - pFix->pParse = pParse; - pFix->zDb = db->aDb[iDb].zName; - pFix->zType = zType; - pFix->pName = pName; - return 1; -} - -/* -** The following set of routines walk through the parse tree and assign -** a specific database to all table references where the database name -** was left unspecified in the original SQL statement. The pFix structure -** must have been initialized by a prior call to sqlite3FixInit(). -** -** These routines are used to make sure that an index, trigger, or -** view in one database does not refer to objects in a different database. -** (Exception: indices, triggers, and views in the TEMP database are -** allowed to refer to anything.) If a reference is explicitly made -** to an object in a different database, an error message is added to -** pParse->zErrMsg and these routines return non-zero. If everything -** checks out, these routines return 0. -*/ -SQLITE_PRIVATE int sqlite3FixSrcList( - DbFixer *pFix, /* Context of the fixation */ - SrcList *pList /* The Source list to check and modify */ -){ - int i; - const char *zDb; - struct SrcList_item *pItem; - - if( NEVER(pList==0) ) return 0; - zDb = pFix->zDb; - for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); - }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ - sqlite3ErrorMsg(pFix->pParse, - "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); - return 1; + case TK_NULL: { + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + int n; + const char *z; + char *zBlob; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + z = &pExpr->u.zToken[2]; + n = sqlite3Strlen30(z) - 1; + assert( z[n]=='\'' ); + zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); + sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); + break; } -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) - if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; - if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; #endif - } - return 0; -} -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) -SQLITE_PRIVATE int sqlite3FixSelect( - DbFixer *pFix, /* Context of the fixation */ - Select *pSelect /* The SELECT statement to be fixed to one database */ -){ - while( pSelect ){ - if( sqlite3FixExprList(pFix, pSelect->pEList) ){ - return 1; + case TK_VARIABLE: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken!=0 ); + assert( pExpr->u.zToken[0]!=0 ); + sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); + if( pExpr->u.zToken[1]!=0 ){ + assert( pExpr->u.zToken[0]=='?' + || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); + sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); + } + break; } - if( sqlite3FixSrcList(pFix, pSelect->pSrc) ){ - return 1; + case TK_REGISTER: { + inReg = pExpr->iTable; + break; } - if( sqlite3FixExpr(pFix, pSelect->pWhere) ){ - return 1; + case TK_AS: { + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + break; } - if( sqlite3FixExpr(pFix, pSelect->pHaving) ){ - return 1; +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + int aff, to_op; + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + aff = sqlite3AffinityType(pExpr->u.zToken); + to_op = aff - SQLITE_AFF_TEXT + OP_ToText; + assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); + assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); + assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); + assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); + assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); + testcase( to_op==OP_ToText ); + testcase( to_op==OP_ToBlob ); + testcase( to_op==OP_ToNumeric ); + testcase( to_op==OP_ToInt ); + testcase( to_op==OP_ToReal ); + if( inReg!=target ){ + sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); + inReg = target; + } + sqlite3VdbeAddOp1(v, to_op, inReg); + testcase( usedAsColumnCache(pParse, inReg, inReg) ); + sqlite3ExprCacheAffinityChange(pParse, inReg, 1); + break; } - pSelect = pSelect->pPrior; - } - return 0; -} -SQLITE_PRIVATE int sqlite3FixExpr( - DbFixer *pFix, /* Context of the fixation */ - Expr *pExpr /* The expression to be fixed to one database */ -){ - while( pExpr ){ - if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ) break; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; - }else{ - if( sqlite3FixExprList(pFix, pExpr->x.pList) ) return 1; +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + assert( TK_LT==OP_Lt ); + assert( TK_LE==OP_Le ); + assert( TK_GT==OP_Gt ); + assert( TK_GE==OP_Ge ); + assert( TK_EQ==OP_Eq ); + assert( TK_NE==OP_Ne ); + testcase( op==TK_LT ); + testcase( op==TK_LE ); + testcase( op==TK_GT ); + testcase( op==TK_GE ); + testcase( op==TK_EQ ); + testcase( op==TK_NE ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; } - if( sqlite3FixExpr(pFix, pExpr->pRight) ){ - return 1; + case TK_IS: + case TK_ISNOT: { + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (op==TK_IS) ? TK_EQ : TK_NE; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; } - pExpr = pExpr->pLeft; - } - return 0; -} -SQLITE_PRIVATE int sqlite3FixExprList( - DbFixer *pFix, /* Context of the fixation */ - ExprList *pList /* The expression to be fixed to one database */ -){ - int i; - struct ExprList_item *pItem; - if( pList==0 ) return 0; - for(i=0, pItem=pList->a; inExpr; i++, pItem++){ - if( sqlite3FixExpr(pFix, pItem->pExpr) ){ - return 1; + case TK_AND: + case TK_OR: + case TK_PLUS: + case TK_STAR: + case TK_MINUS: + case TK_REM: + case TK_BITAND: + case TK_BITOR: + case TK_SLASH: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: { + assert( TK_AND==OP_And ); + assert( TK_OR==OP_Or ); + assert( TK_PLUS==OP_Add ); + assert( TK_MINUS==OP_Subtract ); + assert( TK_REM==OP_Remainder ); + assert( TK_BITAND==OP_BitAnd ); + assert( TK_BITOR==OP_BitOr ); + assert( TK_SLASH==OP_Divide ); + assert( TK_LSHIFT==OP_ShiftLeft ); + assert( TK_RSHIFT==OP_ShiftRight ); + assert( TK_CONCAT==OP_Concat ); + testcase( op==TK_AND ); + testcase( op==TK_OR ); + testcase( op==TK_PLUS ); + testcase( op==TK_MINUS ); + testcase( op==TK_REM ); + testcase( op==TK_BITAND ); + testcase( op==TK_BITOR ); + testcase( op==TK_SLASH ); + testcase( op==TK_LSHIFT ); + testcase( op==TK_RSHIFT ); + testcase( op==TK_CONCAT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; } - } - return 0; -} + case TK_UMINUS: { + Expr *pLeft = pExpr->pLeft; + assert( pLeft ); + if( pLeft->op==TK_INTEGER ){ + codeInteger(pParse, pLeft, 1, target); +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( pLeft->op==TK_FLOAT ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pLeft->u.zToken, 1, target); #endif - -#ifndef SQLITE_OMIT_TRIGGER -SQLITE_PRIVATE int sqlite3FixTriggerStep( - DbFixer *pFix, /* Context of the fixation */ - TriggerStep *pStep /* The trigger step be fixed to one database */ -){ - while( pStep ){ - if( sqlite3FixSelect(pFix, pStep->pSelect) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pStep->pWhere) ){ - return 1; + }else{ + regFree1 = r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); + sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); + testcase( regFree2==0 ); + } + inReg = target; + break; } - if( sqlite3FixExprList(pFix, pStep->pExprList) ){ - return 1; + case TK_BITNOT: + case TK_NOT: { + assert( TK_BITNOT==OP_BitNot ); + assert( TK_NOT==OP_Not ); + testcase( op==TK_BITNOT ); + testcase( op==TK_NOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + inReg = target; + sqlite3VdbeAddOp2(v, op, r1, inReg); + break; } - pStep = pStep->pNext; - } - return 0; -} -#endif + case TK_ISNULL: + case TK_NOTNULL: { + int addr; + assert( TK_ISNULL==OP_IsNull ); + assert( TK_NOTNULL==OP_NotNull ); + testcase( op==TK_ISNULL ); + testcase( op==TK_NOTNULL ); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + addr = sqlite3VdbeAddOp1(v, op, r1); + sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); + sqlite3VdbeJumpHere(v, addr); + break; + } + case TK_AGG_FUNCTION: { + AggInfo *pInfo = pExpr->pAggInfo; + if( pInfo==0 ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); + }else{ + inReg = pInfo->aFunc[pExpr->iAgg].iMem; + } + break; + } + case TK_CONST_FUNC: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + int nFarg; /* Number of function arguments */ + FuncDef *pDef; /* The function definition object */ + int nId; /* Length of the function name in bytes */ + const char *zId; /* The function name */ + int constMask = 0; /* Mask of function arguments that are constant */ + int i; /* Loop counter */ + u8 enc = ENC(db); /* The text encoding used by this database */ + CollSeq *pColl = 0; /* A collating sequence */ -/************** End of attach.c **********************************************/ -/************** Begin file auth.c ********************************************/ -/* -** 2003 January 11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code used to implement the sqlite3_set_authorizer() -** API. This facility is an optional feature of the library. Embedded -** systems that do not need this facility may omit it by recompiling -** the library with -DSQLITE_OMIT_AUTHORIZATION=1 -*/ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + testcase( op==TK_CONST_FUNC ); + testcase( op==TK_FUNCTION ); + if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + nFarg = pFarg ? pFarg->nExpr : 0; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + zId = pExpr->u.zToken; + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); + if( pDef==0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); + break; + } -/* -** All of the code in this file may be omitted by defining a single -** macro. -*/ -#ifndef SQLITE_OMIT_AUTHORIZATION + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evalation of + ** arguments past the first non-NULL argument. + */ + if( pDef->flags & SQLITE_FUNC_COALESCE ){ + int endCoalesce = sqlite3VdbeMakeLabel(v); + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + sqlite3ExprCachePop(pParse, 1); + } + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } -/* -** Set or clear the access authorization function. -** -** The access authorization function is be called during the compilation -** phase to verify that the user has read and/or write access permission on -** various fields of the database. The first argument to the auth function -** is a copy of the 3rd argument to this routine. The second argument -** to the auth function is one of these constants: -** -** SQLITE_CREATE_INDEX -** SQLITE_CREATE_TABLE -** SQLITE_CREATE_TEMP_INDEX -** SQLITE_CREATE_TEMP_TABLE -** SQLITE_CREATE_TEMP_TRIGGER -** SQLITE_CREATE_TEMP_VIEW -** SQLITE_CREATE_TRIGGER -** SQLITE_CREATE_VIEW -** SQLITE_DELETE -** SQLITE_DROP_INDEX -** SQLITE_DROP_TABLE -** SQLITE_DROP_TEMP_INDEX -** SQLITE_DROP_TEMP_TABLE -** SQLITE_DROP_TEMP_TRIGGER -** SQLITE_DROP_TEMP_VIEW -** SQLITE_DROP_TRIGGER -** SQLITE_DROP_VIEW -** SQLITE_INSERT -** SQLITE_PRAGMA -** SQLITE_READ -** SQLITE_SELECT -** SQLITE_TRANSACTION -** SQLITE_UPDATE -** -** The third and fourth arguments to the auth function are the name of -** the table and the column that are being accessed. The auth function -** should return either SQLITE_OK, SQLITE_DENY, or SQLITE_IGNORE. If -** SQLITE_OK is returned, it means that access is allowed. SQLITE_DENY -** means that the SQL statement will never-run - the sqlite3_exec() call -** will return with an error. SQLITE_IGNORE means that the SQL statement -** should run but attempts to read the specified column will return NULL -** and attempts to write the column will be ignored. -** -** Setting the auth function to NULL disables this hook. The default -** setting of the auth function is NULL. -*/ -SQLITE_API int sqlite3_set_authorizer( - sqlite3 *db, - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), - void *pArg -){ - sqlite3_mutex_enter(db->mutex); - db->xAuth = xAuth; - db->pAuthArg = pArg; - sqlite3ExpirePreparedStatements(db); - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; + + if( pFarg ){ + r1 = sqlite3GetTempRange(pParse, nFarg); + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ + sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); + sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ + }else{ + r1 = 0; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Possibly overload the function if the first argument is + ** a virtual table column. + ** + ** For infix functions (LIKE, GLOB, REGEXP, and MATCH) use the + ** second argument, not the first, as the argument to test to + ** see if it is a column in a virtual table. This is done because + ** the left operand of infix functions (the operand we want to + ** control overloading) ends up as the second argument to the + ** function. The expression "A glob B" is equivalent to + ** "glob(B,A). We want to use the A in "A glob B" to test + ** for function overloading. But we use the B term in "glob(B,A)". + */ + if( nFarg>=2 && (pExpr->flags & EP_InfixFunc) ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); + }else if( nFarg>0 ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); + } +#endif + for(i=0; ia[i].pExpr) ){ + constMask |= (1<flags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); + } + } + if( pDef->flags & SQLITE_FUNC_NEEDCOLL ){ + if( !pColl ) pColl = db->pDfltColl; + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, + (char*)pDef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nFarg); + if( nFarg ){ + sqlite3ReleaseTempRange(pParse, r1, nFarg); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: + case TK_SELECT: { + testcase( op==TK_EXISTS ); + testcase( op==TK_SELECT ); + inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + break; + } + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfNull = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + sqlite3VdbeResolveLabel(v, destIfFalse); + sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); + sqlite3VdbeResolveLabel(v, destIfNull); + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pLeft = pExpr->pLeft; + struct ExprList_item *pLItem = pExpr->x.pList->a; + Expr *pRight = pLItem->pExpr; + + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + r3 = sqlite3GetTempReg(pParse); + r4 = sqlite3GetTempReg(pParse); + codeCompare(pParse, pLeft, pRight, OP_Ge, + r1, r2, r3, SQLITE_STOREP2); + pLItem++; + pRight = pLItem->pExpr; + sqlite3ReleaseTempReg(pParse, regFree2); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree2==0 ); + codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); + sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); + sqlite3ReleaseTempReg(pParse, r3); + sqlite3ReleaseTempReg(pParse, r4); + break; + } + case TK_UPLUS: { + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + break; + } + + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + ** + ** The expression is implemented using an OP_Param opcode. The p1 + ** parameter is set to 0 for an old.rowid reference, or to (i+1) + ** to reference another column of the old.* pseudo-table, where + ** i is the index of the column. For a new.rowid reference, p1 is + ** set to (n+1), where n is the number of columns in each pseudo-table. + ** For a reference to any other column in the new.* pseudo-table, p1 + ** is set to (n+2+i), where n and i are as defined previously. For + ** example, if the table on which triggers are being fired is + ** declared as: + ** + ** CREATE TABLE t1(a, b); + ** + ** Then p1 is interpreted as follows: + ** + ** p1==0 -> old.rowid p1==3 -> new.rowid + ** p1==1 -> old.a p1==4 -> new.a + ** p1==2 -> old.b p1==5 -> new.b + */ + Table *pTab = pExpr->pTab; + int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + pExpr->iColumn; + + assert( pExpr->iTable==0 || pExpr->iTable==1 ); + assert( pExpr->iColumn>=-1 && pExpr->iColumnnCol ); + assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey ); + assert( p1>=0 && p1<(pTab->nCol*2+2) ); + + sqlite3VdbeAddOp2(v, OP_Param, p1, target); + VdbeComment((v, "%s.%s -> $%d", + (pExpr->iTable ? "new" : "old"), + (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName), + target + )); + +#ifndef SQLITE_OMIT_FLOATING_POINT + /* If the column has REAL affinity, it may currently be stored as an + ** integer. Use OP_RealAffinity to make sure it is really real. */ + if( pExpr->iColumn>=0 + && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL + ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } +#endif + break; + } + + + /* + ** Form A: + ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form B: + ** CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form A is can be transformed into the equivalent form B as follows: + ** CASE WHEN x=e1 THEN r1 WHEN x=e2 THEN r2 ... + ** WHEN x=eN THEN rN ELSE y END + ** + ** X (if it exists) is in pExpr->pLeft. + ** Y is in pExpr->pRight. The Y is also optional. If there is no + ** ELSE clause and no other term matches, then the result of the + ** exprssion is NULL. + ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. + ** + ** The result of the expression is the Ri for the first matching Ei, + ** or if there is no matching Ei, the ELSE term Y, or if there is + ** no ELSE term, NULL. + */ + default: assert( op==TK_CASE ); { + int endLabel; /* GOTO label for end of CASE stmt */ + int nextCase; /* GOTO label for next WHEN clause */ + int nExpr; /* 2x number of WHEN terms */ + int i; /* Loop counter */ + ExprList *pEList; /* List of WHEN terms */ + struct ExprList_item *aListelem; /* Array of WHEN terms */ + Expr opCompare; /* The X==Ei expression */ + Expr cacheX; /* Cached expression X */ + Expr *pX; /* The X expression */ + Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ + VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); + assert((pExpr->x.pList->nExpr % 2) == 0); + assert(pExpr->x.pList->nExpr > 0); + pEList = pExpr->x.pList; + aListelem = pEList->a; + nExpr = pEList->nExpr; + endLabel = sqlite3VdbeMakeLabel(v); + if( (pX = pExpr->pLeft)!=0 ){ + cacheX = *pX; + testcase( pX->op==TK_COLUMN ); + testcase( pX->op==TK_REGISTER ); + cacheX.iTable = sqlite3ExprCodeTemp(pParse, pX, ®Free1); + testcase( regFree1==0 ); + cacheX.op = TK_REGISTER; + opCompare.op = TK_EQ; + opCompare.pLeft = &cacheX; + pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; + } + for(i=0; iop==TK_COLUMN ); + sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); + testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); + testcase( aListelem[i+1].pExpr->op==TK_REGISTER ); + sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3ExprCachePop(pParse, 1); + sqlite3VdbeResolveLabel(v, nextCase); + } + if( pExpr->pRight ){ + sqlite3ExprCachePush(pParse); + sqlite3ExprCode(pParse, pExpr->pRight, target); + sqlite3ExprCachePop(pParse, 1); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + assert( db->mallocFailed || pParse->nErr>0 + || pParse->iCacheLevel==iCacheLevel ); + sqlite3VdbeResolveLabel(v, endLabel); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + assert( pExpr->affinity==OE_Rollback + || pExpr->affinity==OE_Abort + || pExpr->affinity==OE_Fail + || pExpr->affinity==OE_Ignore + ); + if( !pParse->pTriggerTab ){ + sqlite3ErrorMsg(pParse, + "RAISE() may only be used within a trigger-program"); + return 0; + } + if( pExpr->affinity==OE_Abort ){ + sqlite3MayAbort(pParse); + } + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + if( pExpr->affinity==OE_Ignore ){ + sqlite3VdbeAddOp4( + v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + }else{ + sqlite3HaltConstraint(pParse, pExpr->affinity, pExpr->u.zToken, 0); + } + + break; + } +#endif + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + return inReg; } /* -** Write an error message into pParse->zErrMsg that explains that the -** user-supplied authorization function returned an illegal value. +** Generate code to evaluate an expression and store the results +** into a register. Return the register number where the results +** are stored. +** +** If the register is a temporary register that can be deallocated, +** then write its number into *pReg. If the result register is not +** a temporary, then set *pReg to zero. */ -static void sqliteAuthBadReturnCode(Parse *pParse){ - sqlite3ErrorMsg(pParse, "authorizer malfunction"); - pParse->rc = SQLITE_ERROR; +SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r2==r1 ){ + *pReg = r1; + }else{ + sqlite3ReleaseTempReg(pParse, r1); + *pReg = 0; + } + return r2; } /* -** Invoke the authorization callback for permission to read column zCol from -** table zTab in database zDb. This function assumes that an authorization -** callback has been registered (i.e. that sqlite3.xAuth is not NULL). -** -** If SQLITE_IGNORE is returned and pExpr is not NULL, then pExpr is changed -** to an SQL NULL expression. Otherwise, if pExpr is NULL, then SQLITE_IGNORE -** is treated as SQLITE_DENY. In this case an error is left in pParse. +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. */ -SQLITE_PRIVATE int sqlite3AuthReadCol( - Parse *pParse, /* The parser context */ - const char *zTab, /* Table name */ - const char *zCol, /* Column name */ - int iDb /* Index of containing database. */ -){ - sqlite3 *db = pParse->db; /* Database handle */ - char *zDb = db->aDb[iDb].zName; /* Name of attached database */ - int rc; /* Auth callback return code */ +SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ + int inReg; - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); - if( rc==SQLITE_DENY ){ - if( db->nDb>2 || iDb!=0 ){ - sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); - }else{ - sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol); + assert( target>0 && target<=pParse->nMem ); + if( pExpr && pExpr->op==TK_REGISTER ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); + }else{ + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } - pParse->rc = SQLITE_AUTH; - }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ - sqliteAuthBadReturnCode(pParse); } - return rc; + return target; } /* -** The pExpr should be a TK_COLUMN expression. The table referred to -** is in pTabList or else it is the NEW or OLD table of a trigger. -** Check to see if it is OK to read this particular column. +** Generate code that evalutes the given expression and puts the result +** in register target. ** -** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN -** instruction into a TK_NULL. If the auth function returns SQLITE_DENY, -** then generate an error. +** Also make a copy of the expression results into another "cache" register +** and modify the expression so that the next time it is evaluated, +** the result is a copy of the cache register. +** +** This routine is used for expressions that are used multiple +** times. They are evaluated once and the results of the expression +** are reused. */ -SQLITE_PRIVATE void sqlite3AuthRead( - Parse *pParse, /* The parser context */ - Expr *pExpr, /* The expression to check authorization on */ - Schema *pSchema, /* The schema of the expression */ - SrcList *pTabList /* All table that pExpr might refer to */ -){ - sqlite3 *db = pParse->db; - Table *pTab = 0; /* The table being read */ - const char *zCol; /* Name of the column of the table */ - int iSrc; /* Index in pTabList->a[] of table being read */ - int iDb; /* The index of the database the expression refers to */ - int iCol; /* Index of column in table */ - - if( db->xAuth==0 ) return; - iDb = sqlite3SchemaToIndex(pParse->db, pSchema); - if( iDb<0 ){ - /* An attempt to read a column out of a subquery or other - ** temporary table. */ - return; +SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; + int inReg; + inReg = sqlite3ExprCode(pParse, pExpr, target); + assert( target>0 ); + /* This routine is called for terms to INSERT or UPDATE. And the only + ** other place where expressions can be converted into TK_REGISTER is + ** in WHERE clause processing. So as currently implemented, there is + ** no way for a TK_REGISTER to exist here. But it seems prudent to + ** keep the ALWAYS() in case the conditions above change with future + ** modifications or enhancements. */ + if( ALWAYS(pExpr->op!=TK_REGISTER) ){ + int iMem; + iMem = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, inReg, iMem); + pExpr->iTable = iMem; + pExpr->op2 = pExpr->op; + pExpr->op = TK_REGISTER; } + return inReg; +} - assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); - if( pExpr->op==TK_TRIGGER ){ - pTab = pParse->pTriggerTab; - }else{ - assert( pTabList ); - for(iSrc=0; ALWAYS(iSrcnSrc); iSrc++){ - if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ - pTab = pTabList->a[iSrc].pTab; - break; - } - } +/* +** Return TRUE if pExpr is an constant expression that is appropriate +** for factoring out of a loop. Appropriate expressions are: +** +** * Any expression that evaluates to two or more opcodes. +** +** * Any OP_Integer, OP_Real, OP_String, OP_Blob, OP_Null, +** or OP_Variable that does not need to be placed in a +** specific register. +** +** There is no point in factoring out single-instruction constant +** expressions that need to be placed in a particular register. +** We could factor them out, but then we would end up adding an +** OP_SCopy instruction to move the value into the correct register +** later. We might as well just use the original instruction and +** avoid the OP_SCopy. +*/ +static int isAppropriateForFactoring(Expr *p){ + if( !sqlite3ExprIsConstantNotJoin(p) ){ + return 0; /* Only constant expressions are appropriate for factoring */ } - iCol = pExpr->iColumn; - if( NEVER(pTab==0) ) return; - - if( iCol>=0 ){ - assert( iColnCol ); - zCol = pTab->aCol[iCol].zName; - }else if( pTab->iPKey>=0 ){ - assert( pTab->iPKeynCol ); - zCol = pTab->aCol[pTab->iPKey].zName; - }else{ - zCol = "ROWID"; + if( (p->flags & EP_FixedDest)==0 ){ + return 1; /* Any constant without a fixed destination is appropriate */ } - assert( iDb>=0 && iDbnDb ); - if( SQLITE_IGNORE==sqlite3AuthReadCol(pParse, pTab->zName, zCol, iDb) ){ - pExpr->op = TK_NULL; + while( p->op==TK_UPLUS ) p = p->pLeft; + switch( p->op ){ +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: +#endif + case TK_VARIABLE: + case TK_INTEGER: + case TK_FLOAT: + case TK_NULL: + case TK_STRING: { + testcase( p->op==TK_BLOB ); + testcase( p->op==TK_VARIABLE ); + testcase( p->op==TK_INTEGER ); + testcase( p->op==TK_FLOAT ); + testcase( p->op==TK_NULL ); + testcase( p->op==TK_STRING ); + /* Single-instruction constants with a fixed destination are + ** better done in-line. If we factor them, they will just end + ** up generating an OP_SCopy to move the value to the destination + ** register. */ + return 0; + } + case TK_UMINUS: { + if( p->pLeft->op==TK_FLOAT || p->pLeft->op==TK_INTEGER ){ + return 0; + } + break; + } + default: { + break; + } } + return 1; } /* -** Do an authorization check using the code and arguments given. Return -** either SQLITE_OK (zero) or SQLITE_IGNORE or SQLITE_DENY. If SQLITE_DENY -** is returned, then the error count and error message in pParse are -** modified appropriately. +** If pExpr is a constant expression that is appropriate for +** factoring out of a loop, then evaluate the expression +** into a register and convert the expression into a TK_REGISTER +** expression. */ -SQLITE_PRIVATE int sqlite3AuthCheck( - Parse *pParse, - int code, - const char *zArg1, - const char *zArg2, - const char *zArg3 -){ - sqlite3 *db = pParse->db; - int rc; - - /* Don't do any authorization checks if the database is initialising - ** or if the parser is being invoked from within sqlite3_declare_vtab. - */ - if( db->init.busy || IN_DECLARE_VTAB ){ - return SQLITE_OK; - } - - if( db->xAuth==0 ){ - return SQLITE_OK; +static int evalConstExpr(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + switch( pExpr->op ){ + case TK_IN: + case TK_REGISTER: { + return WRC_Prune; + } + case TK_FUNCTION: + case TK_AGG_FUNCTION: + case TK_CONST_FUNC: { + /* The arguments to a function have a fixed destination. + ** Mark them this way to avoid generated unneeded OP_SCopy + ** instructions. + */ + ExprList *pList = pExpr->x.pList; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( pList ){ + int i = pList->nExpr; + struct ExprList_item *pItem = pList->a; + for(; i>0; i--, pItem++){ + if( ALWAYS(pItem->pExpr) ) pItem->pExpr->flags |= EP_FixedDest; + } + } + break; + } } - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); - if( rc==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized"); - pParse->rc = SQLITE_AUTH; - }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){ - rc = SQLITE_DENY; - sqliteAuthBadReturnCode(pParse); + if( isAppropriateForFactoring(pExpr) ){ + int r1 = ++pParse->nMem; + int r2; + r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); + pExpr->op2 = pExpr->op; + pExpr->op = TK_REGISTER; + pExpr->iTable = r2; + return WRC_Prune; } - return rc; + return WRC_Continue; } /* -** Push an authorization context. After this routine is called, the -** zArg3 argument to authorization callbacks will be zContext until -** popped. Or if pParse==0, this routine is a no-op. +** Preevaluate constant subexpressions within pExpr and store the +** results in registers. Modify pExpr so that the constant subexpresions +** are TK_REGISTER opcodes that refer to the precomputed values. +** +** This routine is a no-op if the jump to the cookie-check code has +** already occur. Since the cookie-check jump is generated prior to +** any other serious processing, this check ensures that there is no +** way to accidently bypass the constant initializations. +** +** This routine is also a no-op if the SQLITE_FactorOutConst optimization +** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) +** interface. This allows test logic to verify that the same answer is +** obtained for queries regardless of whether or not constants are +** precomputed into registers or if they are inserted in-line. */ -SQLITE_PRIVATE void sqlite3AuthContextPush( - Parse *pParse, - AuthContext *pContext, - const char *zContext -){ - assert( pParse ); - pContext->pParse = pParse; - pContext->zAuthContext = pParse->zAuthContext; - pParse->zAuthContext = zContext; +SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ + Walker w; + if( pParse->cookieGoto ) return; + if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; + w.xExprCallback = evalConstExpr; + w.xSelectCallback = 0; + w.pParse = pParse; + sqlite3WalkExpr(&w, pExpr); } + /* -** Pop an authorization context that was previously pushed -** by sqlite3AuthContextPush +** Generate code that pushes the value of every element of the given +** expression list into a sequence of registers beginning at target. +** +** Return the number of elements evaluated. */ -SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ - if( pContext->pParse ){ - pContext->pParse->zAuthContext = pContext->zAuthContext; - pContext->pParse = 0; +SQLITE_PRIVATE int sqlite3ExprCodeExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* The expression list to be coded */ + int target, /* Where to write results */ + int doHardCopy /* Make a hard copy of every element */ +){ + struct ExprList_item *pItem; + int i, n; + assert( pList!=0 ); + assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ + n = pList->nExpr; + for(pItem=pList->a, i=0; ipExpr; + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + sqlite3VdbeAddOp2(pParse->pVdbe, doHardCopy ? OP_Copy : OP_SCopy, + inReg, target+i); + } } + return n; } -#endif /* SQLITE_OMIT_AUTHORIZATION */ - -/************** End of auth.c ************************************************/ -/************** Begin file build.c *******************************************/ /* -** 2001 September 15 +** Generate code for a BETWEEN operator. ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** x BETWEEN y AND z ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** The above is equivalent to ** -************************************************************************* -** This file contains C code routines that are called by the SQLite parser -** when syntax rules are reduced. The routines in this file handle the -** following kinds of SQL syntax: +** x>=y AND x<=z ** -** CREATE TABLE -** DROP TABLE -** CREATE INDEX -** DROP INDEX -** creating ID lists -** BEGIN TRANSACTION -** COMMIT -** ROLLBACK +** Code it as such, taking care to do the common subexpression +** elementation of x. */ +static void exprCodeBetween( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The BETWEEN expression */ + int dest, /* Jump here if the jump is taken */ + int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int jumpIfNull /* Take the jump if the BETWEEN is NULL */ +){ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr compLeft; /* The x>=y term */ + Expr compRight; /* The x<=z term */ + Expr exprX; /* The x subexpression */ + int regFree1 = 0; /* Temporary use register */ -/* -** This routine is called when a new SQL statement is beginning to -** be parsed. Initialize the pParse structure as needed. -*/ -SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){ - pParse->explain = (u8)explainFlag; - pParse->nVar = 0; -} + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + exprX = *pExpr->pLeft; + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = &exprX; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = &exprX; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); + exprX.op = TK_REGISTER; + if( jumpIfTrue ){ + sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + }else{ + sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + } + sqlite3ReleaseTempReg(pParse, regFree1); -#ifndef SQLITE_OMIT_SHARED_CACHE -/* -** The TableLock structure is only used by the sqlite3TableLock() and -** codeTableLocks() functions. -*/ -struct TableLock { - int iDb; /* The database containing the table to be locked */ - int iTab; /* The root page of the table to be locked */ - u8 isWriteLock; /* True for write lock. False for a read lock */ - const char *zName; /* Name of the table */ -}; + /* Ensure adequate test coverage */ + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); +} /* -** Record the fact that we want to lock a table at run-time. +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is true but execution +** continues straight thru if the expression is false. ** -** The table to be locked has root page iTab and is found in database iDb. -** A read or a write lock can be taken depending on isWritelock. +** If the expression evaluates to NULL (neither true nor false), then +** take the jump if the jumpIfNull flag is SQLITE_JUMPIFNULL. ** -** This routine just records the fact that the lock is desired. The -** code to make the lock occur is generated by a later call to -** codeTableLocks() which occurs during sqlite3FinishCoding(). +** This code depends on the fact that certain token values (ex: TK_EQ) +** are the same as opcode values (ex: OP_Eq) that implement the corresponding +** operation. Special comments in vdbe.c and the mkopcodeh.awk script in +** the make process cause these values to align. Assert()s in the code +** below verify that the numbers are aligned correctly. */ -SQLITE_PRIVATE void sqlite3TableLock( - Parse *pParse, /* Parsing context */ - int iDb, /* Index of the database containing the table to lock */ - int iTab, /* Root page number of the table to be locked */ - u8 isWriteLock, /* True for a write lock */ - const char *zName /* Name of the table to be locked */ -){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - int i; - int nBytes; - TableLock *p; - assert( iDb>=0 ); +SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; - for(i=0; inTableLock; i++){ - p = &pToplevel->aTableLock[i]; - if( p->iDb==iDb && p->iTab==iTab ){ - p->isWriteLock = (p->isWriteLock || isWriteLock); - return; + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(pExpr==0) ) return; /* No way this can happen */ + op = pExpr->op; + switch( op ){ + case TK_AND: { + int d2 = sqlite3VdbeMakeLabel(v); + testcase( jumpIfNull==0 ); + sqlite3ExprCachePush(pParse); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + sqlite3ExprCachePop(pParse, 1); + break; + } + case TK_OR: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + assert( TK_LT==OP_Lt ); + assert( TK_LE==OP_Le ); + assert( TK_GT==OP_Gt ); + assert( TK_GE==OP_Ge ); + assert( TK_EQ==OP_Eq ); + assert( TK_NE==OP_Ne ); + testcase( op==TK_LT ); + testcase( op==TK_LE ); + testcase( op==TK_GT ); + testcase( op==TK_GE ); + testcase( op==TK_EQ ); + testcase( op==TK_NE ); + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (op==TK_IS) ? TK_EQ : TK_NE; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + assert( TK_ISNULL==OP_IsNull ); + assert( TK_NOTNULL==OP_NotNull ); + testcase( op==TK_ISNULL ); + testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfNull = jumpIfNull ? dest : destIfFalse; + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeResolveLabel(v, destIfFalse); + break; + } +#endif + default: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + break; } } - - nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); - pToplevel->aTableLock = - sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); - if( pToplevel->aTableLock ){ - p = &pToplevel->aTableLock[pToplevel->nTableLock++]; - p->iDb = iDb; - p->iTab = iTab; - p->isWriteLock = isWriteLock; - p->zName = zName; - }else{ - pToplevel->nTableLock = 0; - pToplevel->db->mallocFailed = 1; - } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); } /* -** Code an OP_TableLock instruction for each table locked by the -** statement (configured by calls to sqlite3TableLock()). +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is false but execution +** continues straight thru if the expression is true. +** +** If the expression evaluates to NULL (neither true nor false) then +** jump if jumpIfNull is SQLITE_JUMPIFNULL or fall through if jumpIfNull +** is 0. */ -static void codeTableLocks(Parse *pParse){ - int i; - Vdbe *pVdbe; - - pVdbe = sqlite3GetVdbe(pParse); - assert( pVdbe!=0 ); /* sqlite3GetVdbe cannot fail: VDBE already allocated */ +SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; - for(i=0; inTableLock; i++){ - TableLock *p = &pParse->aTableLock[i]; - int p1 = p->iDb; - sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, - p->zName, P4_STATIC); - } -} -#else - #define codeTableLocks(x) -#endif - -/* -** This routine is called after a single SQL statement has been -** parsed and a VDBE program to execute that statement has been -** prepared. This routine puts the finishing touches on the -** VDBE program and resets the pParse structure for the next -** parse. -** -** Note that if an error occurred, it might be the case that -** no VDBE code was generated. -*/ -SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ - sqlite3 *db; - Vdbe *v; + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( pExpr==0 ) return; - db = pParse->db; - if( db->mallocFailed ) return; - if( pParse->nested ) return; - if( pParse->nErr ) return; + /* The value of pExpr->op and op are related as follows: + ** + ** pExpr->op op + ** --------- ---------- + ** TK_ISNULL OP_NotNull + ** TK_NOTNULL OP_IsNull + ** TK_NE OP_Eq + ** TK_EQ OP_Ne + ** TK_GT OP_Le + ** TK_LE OP_Gt + ** TK_GE OP_Lt + ** TK_LT OP_Ge + ** + ** For other values of pExpr->op, op is undefined and unused. + ** The value of TK_ and OP_ constants are arranged such that we + ** can compute the mapping above using the following expression. + ** Assert()s verify that the computation is correct. + */ + op = ((pExpr->op+(TK_ISNULL&1))^1)-(TK_ISNULL&1); - /* Begin by generating some termination code at the end of the - ** vdbe program + /* Verify correct alignment of TK_ and OP_ constants */ - v = sqlite3GetVdbe(pParse); - assert( !pParse->isMultiWrite - || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); - if( v ){ - sqlite3VdbeAddOp0(v, OP_Halt); + assert( pExpr->op!=TK_ISNULL || op==OP_NotNull ); + assert( pExpr->op!=TK_NOTNULL || op==OP_IsNull ); + assert( pExpr->op!=TK_NE || op==OP_Eq ); + assert( pExpr->op!=TK_EQ || op==OP_Ne ); + assert( pExpr->op!=TK_LT || op==OP_Ge ); + assert( pExpr->op!=TK_LE || op==OP_Gt ); + assert( pExpr->op!=TK_GT || op==OP_Le ); + assert( pExpr->op!=TK_GE || op==OP_Lt ); - /* The cookie mask contains one bit for each database file open. - ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are - ** set for each database that is used. Generate code to start a - ** transaction on each used database and to verify the schema cookie - ** on each used database. - */ - if( pParse->cookieGoto>0 ){ - u32 mask; - int iDb; - sqlite3VdbeJumpHere(v, pParse->cookieGoto-1); - for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ - if( (mask & pParse->cookieMask)==0 ) continue; - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v,OP_Transaction, iDb, (mask & pParse->writeMask)!=0); - if( db->init.busy==0 ){ - sqlite3VdbeAddOp2(v,OP_VerifyCookie, iDb, pParse->cookieValue[iDb]); - } - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - { - int i; - for(i=0; inVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + switch( pExpr->op ){ + case TK_AND: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + break; + } + case TK_OR: { + int d2 = sqlite3VdbeMakeLabel(v); + testcase( jumpIfNull==0 ); + sqlite3ExprCachePush(pParse); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + sqlite3ExprCachePop(pParse, 1); + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + testcase( op==TK_LT ); + testcase( op==TK_LE ); + testcase( op==TK_GT ); + testcase( op==TK_GE ); + testcase( op==TK_EQ ); + testcase( op==TK_NE ); + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_IS: + case TK_ISNOT: { + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + testcase( op==TK_ISNULL ); + testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + if( jumpIfNull ){ + sqlite3ExprCodeIN(pParse, pExpr, dest, dest); + }else{ + int destIfNull = sqlite3VdbeMakeLabel(v); + sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); + sqlite3VdbeResolveLabel(v, destIfNull); } -#endif - - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the - ** shared-cache feature is enabled. - */ - codeTableLocks(pParse); - - /* Initialize any AUTOINCREMENT data structures required. - */ - sqlite3AutoincrementBegin(pParse); - - /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pParse->cookieGoto); + break; } - } - - - /* Get the VDBE program ready for execution - */ - if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ -#ifdef SQLITE_DEBUG - FILE *trace = (db->flags & SQLITE_VdbeTrace)!=0 ? stdout : 0; - sqlite3VdbeTrace(v, trace); #endif - assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ - /* A minimum of one cursor is required if autoincrement is used - * See ticket [a696379c1f08866] */ - if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse->nVar, pParse->nMem, - pParse->nTab, pParse->nMaxArg, pParse->explain, - pParse->isMultiWrite && pParse->mayAbort); - pParse->rc = SQLITE_DONE; - pParse->colNamesSet = 0; - }else{ - pParse->rc = SQLITE_ERROR; - } - pParse->nTab = 0; - pParse->nMem = 0; - pParse->nSet = 0; - pParse->nVar = 0; - pParse->cookieMask = 0; - pParse->cookieGoto = 0; -} - -/* -** Run the parser and code generator recursively in order to generate -** code for the SQL statement given onto the end of the pParse context -** currently under construction. When the parser is run recursively -** this way, the final OP_Halt is not appended and other initialization -** and finalization steps are omitted because those are handling by the -** outermost parser. -** -** Not everything is nestable. This facility is designed to permit -** INSERT, UPDATE, and DELETE operations against SQLITE_MASTER. Use -** care if you decide to try to use this routine for some other purposes. -*/ -SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ - va_list ap; - char *zSql; - char *zErrMsg = 0; - sqlite3 *db = pParse->db; -# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) - char saveBuf[SAVE_SZ]; - - if( pParse->nErr ) return; - assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ - va_start(ap, zFormat); - zSql = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); - if( zSql==0 ){ - return; /* A malloc must have failed */ + default: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + break; + } } - pParse->nested++; - memcpy(saveBuf, &pParse->nVar, SAVE_SZ); - memset(&pParse->nVar, 0, SAVE_SZ); - sqlite3RunParser(pParse, zSql, &zErrMsg); - sqlite3DbFree(db, zErrMsg); - sqlite3DbFree(db, zSql); - memcpy(&pParse->nVar, saveBuf, SAVE_SZ); - pParse->nested--; + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); } /* -** Locate the in-memory structure that describes a particular database -** table given the name of that table and (optionally) the name of the -** database containing the table. Return NULL if not found. -** -** If zDatabase is 0, all databases are searched for the table and the -** first matching table is returned. (No checking for duplicate table -** names is done.) The search order is TEMP first, then MAIN, then any -** auxiliary databases added using the ATTACH command. +** Do a deep comparison of two expression trees. Return 0 if the two +** expressions are completely identical. Return 1 if they differ only +** by a COLLATE operator at the top level. Return 2 if there are differences +** other than the top-level COLLATE operator. ** -** See also sqlite3LocateTable(). +** Sometimes this routine will return 2 even if the two expressions +** really are equivalent. If we cannot prove that the expressions are +** identical, we return 2 just to be safe. So if this routine +** returns 2, then you do not really know for certain if the two +** expressions are the same. But if you get a 0 or 1 return, then you +** can be sure the expressions are the same. In the places where +** this routine is used, it does not hurt to get an extra 2 - that +** just might result in some slightly slower code. But returning +** an incorrect 0 or 1 could lead to a malfunction. */ -SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ - Table *p = 0; - int i; - int nName; - assert( zName!=0 ); - nName = sqlite3Strlen30(zName); - for(i=OMIT_TEMPDB; inDb; i++){ - int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); - if( p ) break; +SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ + if( pA==0||pB==0 ){ + return pB==pA ? 0 : 2; } - return p; -} - -/* -** Locate the in-memory structure that describes a particular database -** table given the name of that table and (optionally) the name of the -** database containing the table. Return NULL if not found. Also leave an -** error message in pParse->zErrMsg. -** -** The difference between this routine and sqlite3FindTable() is that this -** routine leaves an error message in pParse->zErrMsg where -** sqlite3FindTable() does not. -*/ -SQLITE_PRIVATE Table *sqlite3LocateTable( - Parse *pParse, /* context in which to report errors */ - int isView, /* True if looking for a VIEW rather than a TABLE */ - const char *zName, /* Name of the table we are looking for */ - const char *zDbase /* Name of the database. Might be NULL */ -){ - Table *p; - - /* Read the database schema. If an error occurs, leave an error message - ** and code in pParse and return NULL. */ - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - return 0; + assert( !ExprHasAnyProperty(pA, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasAnyProperty(pB, EP_TokenOnly|EP_Reduced) ); + if( ExprHasProperty(pA, EP_xIsSelect) || ExprHasProperty(pB, EP_xIsSelect) ){ + return 2; } - - p = sqlite3FindTable(pParse->db, zName, zDbase); - if( p==0 ){ - const char *zMsg = isView ? "no such view" : "no such table"; - if( zDbase ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); - }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; + if( pA->op!=pB->op ) return 2; + if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; + if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; + if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2; + if( pA->iTable!=pB->iTable || pA->iColumn!=pB->iColumn ) return 2; + if( ExprHasProperty(pA, EP_IntValue) ){ + if( !ExprHasProperty(pB, EP_IntValue) || pA->u.iValue!=pB->u.iValue ){ + return 2; + } + }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ + if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ){ + return 2; } - pParse->checkSchema = 1; } - return p; + if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; + if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; + return 0; } /* -** Locate the in-memory structure that describes -** a particular index given the name of that index -** and the name of the database that contains the index. -** Return NULL if not found. +** Compare two ExprList objects. Return 0 if they are identical and +** non-zero if they differ in any way. ** -** If zDatabase is 0, all databases are searched for the -** table and the first matching index is returned. (No checking -** for duplicate index names is done.) The search order is -** TEMP first, then MAIN, then any auxiliary databases added -** using the ATTACH command. +** This routine might return non-zero for equivalent ExprLists. The +** only consequence will be disabled optimizations. But this routine +** must never return 0 if the two ExprList objects are different, or +** a malfunction will result. +** +** Two NULL pointers are considered to be the same. But a NULL pointer +** always differs from a non-NULL pointer. */ -SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ - Index *p = 0; +SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB){ int i; - int nName = sqlite3Strlen30(zName); - for(i=OMIT_TEMPDB; inDb; i++){ - int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - Schema *pSchema = db->aDb[j].pSchema; - assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; - p = sqlite3HashFind(&pSchema->idxHash, zName, nName); - if( p ) break; - } - return p; -} - -/* -** Reclaim the memory used by an index -*/ -static void freeIndex(sqlite3 *db, Index *p){ -#ifndef SQLITE_OMIT_ANALYZE - sqlite3DeleteIndexSamples(db, p); -#endif - sqlite3DbFree(db, p->zColAff); - sqlite3DbFree(db, p); -} - -/* -** For the index called zIdxName which is found in the database iDb, -** unlike that index from its Table then remove the index from -** the index hash table and free all memory structures associated -** with the index. -*/ -SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ - Index *pIndex; - int len; - Hash *pHash = &db->aDb[iDb].pSchema->idxHash; - - len = sqlite3Strlen30(zIdxName); - pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); - if( pIndex ){ - if( pIndex->pTable->pIndex==pIndex ){ - pIndex->pTable->pIndex = pIndex->pNext; - }else{ - Index *p; - /* Justification of ALWAYS(); The index must be on the list of - ** indices. */ - p = pIndex->pTable->pIndex; - while( ALWAYS(p) && p->pNext!=pIndex ){ p = p->pNext; } - if( ALWAYS(p && p->pNext==pIndex) ){ - p->pNext = pIndex->pNext; - } - } - freeIndex(db, pIndex); + if( pA==0 && pB==0 ) return 0; + if( pA==0 || pB==0 ) return 1; + if( pA->nExpr!=pB->nExpr ) return 1; + for(i=0; inExpr; i++){ + Expr *pExprA = pA->a[i].pExpr; + Expr *pExprB = pB->a[i].pExpr; + if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; + if( sqlite3ExprCompare(pExprA, pExprB) ) return 1; } - db->flags |= SQLITE_InternChanges; + return 0; } /* -** Erase all schema information from the in-memory hash tables of -** a single database. This routine is called to reclaim memory -** before the database closes. It is also called during a rollback -** if there were schema changes during the transaction or if a -** schema-cookie mismatch occurs. -** -** If iDb==0 then reset the internal schema tables for all database -** files. If iDb>=1 then reset the internal schema for only the -** single file indicated. +** Add a new element to the pAggInfo->aCol[] array. Return the index of +** the new element. Return a negative number if malloc fails. */ -SQLITE_PRIVATE void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ - int i, j; - assert( iDb>=0 && iDbnDb ); - - if( iDb==0 ){ - sqlite3BtreeEnterAll(db); - } - for(i=iDb; inDb; i++){ - Db *pDb = &db->aDb[i]; - if( pDb->pSchema ){ - assert(i==1 || (pDb->pBt && sqlite3BtreeHoldsMutex(pDb->pBt))); - sqlite3SchemaFree(pDb->pSchema); - } - if( iDb>0 ) return; - } - assert( iDb==0 ); - db->flags &= ~SQLITE_InternChanges; - sqlite3VtabUnlockList(db); - sqlite3BtreeLeaveAll(db); - - /* If one or more of the auxiliary database files has been closed, - ** then remove them from the auxiliary database list. We take the - ** opportunity to do this here since we have just deleted all of the - ** schema hash tables and therefore do not have to make any changes - ** to any of those tables. - */ - for(i=j=2; inDb; i++){ - struct Db *pDb = &db->aDb[i]; - if( pDb->pBt==0 ){ - sqlite3DbFree(db, pDb->zName); - pDb->zName = 0; - continue; - } - if( jaDb[j] = db->aDb[i]; - } - j++; - } - memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); - db->nDb = j; - if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ - memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); - sqlite3DbFree(db, db->aDb); - db->aDb = db->aDbStatic; - } -} +static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aCol = sqlite3ArrayAllocate( + db, + pInfo->aCol, + sizeof(pInfo->aCol[0]), + 3, + &pInfo->nColumn, + &pInfo->nColumnAlloc, + &i + ); + return i; +} /* -** This routine is called when a commit occurs. +** Add a new element to the pAggInfo->aFunc[] array. Return the index of +** the new element. Return a negative number if malloc fails. */ -SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ - db->flags &= ~SQLITE_InternChanges; -} +static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aFunc = sqlite3ArrayAllocate( + db, + pInfo->aFunc, + sizeof(pInfo->aFunc[0]), + 3, + &pInfo->nFunc, + &pInfo->nFuncAlloc, + &i + ); + return i; +} /* -** Delete memory allocated for the column names of a table or view (the -** Table.aCol[] array). +** This is the xExprCallback for a tree walker. It is used to +** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates +** for additional information. */ -static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ +static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ int i; - Column *pCol; - assert( pTable!=0 ); - if( (pCol = pTable->aCol)!=0 ){ - for(i=0; inCol; i++, pCol++){ - sqlite3DbFree(db, pCol->zName); - sqlite3ExprDelete(db, pCol->pDflt); - sqlite3DbFree(db, pCol->zDflt); - sqlite3DbFree(db, pCol->zType); - sqlite3DbFree(db, pCol->zColl); + NameContext *pNC = pWalker->u.pNC; + Parse *pParse = pNC->pParse; + SrcList *pSrcList = pNC->pSrcList; + AggInfo *pAggInfo = pNC->pAggInfo; + + switch( pExpr->op ){ + case TK_AGG_COLUMN: + case TK_COLUMN: { + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); + /* Check to see if the column is in one of the tables in the FROM + ** clause of the aggregate query */ + if( ALWAYS(pSrcList!=0) ){ + struct SrcList_item *pItem = pSrcList->a; + for(i=0; inSrc; i++, pItem++){ + struct AggInfo_col *pCol; + assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + if( pExpr->iTable==pItem->iCursor ){ + /* If we reach this point, it means that pExpr refers to a table + ** that is in the FROM clause of the aggregate query. + ** + ** Make an entry for the column in pAggInfo->aCol[] if there + ** is not an entry there already. + */ + int k; + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable && + pCol->iColumn==pExpr->iColumn ){ + break; + } + } + if( (k>=pAggInfo->nColumn) + && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 + ){ + pCol = &pAggInfo->aCol[k]; + pCol->pTab = pExpr->pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iMem = ++pParse->nMem; + pCol->iSorterColumn = -1; + pCol->pExpr = pExpr; + if( pAggInfo->pGroupBy ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && + pE->iColumn==pExpr->iColumn ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } + } + /* There is now an entry for pExpr in pAggInfo->aCol[] (either + ** because it was there before or because we just created it). + ** Convert the pExpr to be a TK_AGG_COLUMN referring to that + ** pAggInfo->aCol[] entry. + */ + ExprSetIrreducible(pExpr); + pExpr->pAggInfo = pAggInfo; + pExpr->op = TK_AGG_COLUMN; + pExpr->iAgg = (i16)k; + break; + } /* endif pExpr->iTable==pItem->iCursor */ + } /* end loop over pSrcList */ + } + return WRC_Prune; + } + case TK_AGG_FUNCTION: { + /* The pNC->nDepth==0 test causes aggregate functions in subqueries + ** to be ignored */ + if( pNC->nDepth==0 ){ + /* Check to see if pExpr is a duplicate of another aggregate + ** function that is already in the pAggInfo structure + */ + struct AggInfo_func *pItem = pAggInfo->aFunc; + for(i=0; inFunc; i++, pItem++){ + if( sqlite3ExprCompare(pItem->pExpr, pExpr)==0 ){ + break; + } + } + if( i>=pAggInfo->nFunc ){ + /* pExpr is original. Make a new entry in pAggInfo->aFunc[] + */ + u8 enc = ENC(pParse->db); + i = addAggInfoFunc(pParse->db, pAggInfo); + if( i>=0 ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pItem = &pAggInfo->aFunc[i]; + pItem->pExpr = pExpr; + pItem->iMem = ++pParse->nMem; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pItem->pFunc = sqlite3FindFunction(pParse->db, + pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), + pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); + if( pExpr->flags & EP_Distinct ){ + pItem->iDistinct = pParse->nTab++; + }else{ + pItem->iDistinct = -1; + } + } + } + /* Make pExpr point to the appropriate pAggInfo->aFunc[] entry + */ + assert( !ExprHasAnyProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + ExprSetIrreducible(pExpr); + pExpr->iAgg = (i16)i; + pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + } } - sqlite3DbFree(db, pTable->aCol); } + return WRC_Continue; } - -/* -** Remove the memory data structures associated with the given -** Table. No changes are made to disk by this routine. -** -** This routine just deletes the data structure. It does not unlink -** the table data structure from the hash table. But it does destroy -** memory structures of the indices and foreign keys associated with -** the table. -*/ -SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ - Index *pIndex, *pNext; - - assert( !pTable || pTable->nRef>0 ); - - /* Do not delete the table until the reference count reaches zero. */ - if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - - /* Delete all indices associated with this table. */ - for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ - pNext = pIndex->pNext; - assert( pIndex->pSchema==pTable->pSchema ); - if( !db || db->pnBytesFreed==0 ){ - char *zName = pIndex->zName; - TESTONLY ( Index *pOld = ) sqlite3HashInsert( - &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 - ); - assert( pOld==pIndex || pOld==0 ); - } - freeIndex(db, pIndex); +static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ + NameContext *pNC = pWalker->u.pNC; + if( pNC->nDepth==0 ){ + pNC->nDepth++; + sqlite3WalkSelect(pWalker, pSelect); + pNC->nDepth--; + return WRC_Prune; + }else{ + return WRC_Continue; } - - /* Delete any foreign keys attached to this table. */ - sqlite3FkDelete(db, pTable); - - /* Delete the Table structure itself. - */ - sqliteDeleteColumnNames(db, pTable); - sqlite3DbFree(db, pTable->zName); - sqlite3DbFree(db, pTable->zColAff); - sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(db, pTable->pCheck); -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3VtabClear(db, pTable); -#endif - sqlite3DbFree(db, pTable); } /* -** Unlink the given table from the hash tables and the delete the -** table structure with all its indices and foreign keys. +** Analyze the given expression looking for aggregate functions and +** for variables that need to be added to the pParse->aAgg[] array. +** Make additional entries to the pParse->aAgg[] array as necessary. +** +** This routine should only be called after the expression has been +** analyzed by sqlite3ResolveExprNames(). */ -SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ - Table *p; - Db *pDb; - - assert( db!=0 ); - assert( iDb>=0 && iDbnDb ); - assert( zTabName ); - testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ - pDb = &db->aDb[iDb]; - p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, - sqlite3Strlen30(zTabName),0); - sqlite3DeleteTable(db, p); - db->flags |= SQLITE_InternChanges; +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ + Walker w; + w.xExprCallback = analyzeAggregate; + w.xSelectCallback = analyzeAggregatesInSelect; + w.u.pNC = pNC; + assert( pNC->pSrcList!=0 ); + sqlite3WalkExpr(&w, pExpr); } /* -** Given a token, return a string that consists of the text of that -** token. Space to hold the returned string -** is obtained from sqliteMalloc() and must be freed by the calling -** function. -** -** Any quotation marks (ex: "name", 'name', [name], or `name`) that -** surround the body of the token are removed. +** Call sqlite3ExprAnalyzeAggregates() for every expression in an +** expression list. Return the number of errors. ** -** Tokens are often just pointers into the original SQL text and so -** are not \000 terminated and are not persistent. The returned string -** is \000 terminated and is persistent. +** If an error is found, the analysis is cut short. */ -SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ - char *zName; - if( pName ){ - zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n); - sqlite3Dequote(zName); - }else{ - zName = 0; +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ + struct ExprList_item *pItem; + int i; + if( pList ){ + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); + } } - return zName; } /* -** Open the sqlite_master table stored in database number iDb for -** writing. The table is opened using cursor 0. +** Allocate a single new register for use to hold some intermediate result. */ -SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *p, int iDb){ - Vdbe *v = sqlite3GetVdbe(p); - sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, MASTER_ROOT, iDb); - sqlite3VdbeChangeP4(v, -1, (char *)5, P4_INT32); /* 5 column table */ - if( p->nTab==0 ){ - p->nTab = 1; +SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ + if( pParse->nTempReg==0 ){ + return ++pParse->nMem; } + return pParse->aTempReg[--pParse->nTempReg]; } /* -** Parameter zName points to a nul-terminated buffer containing the name -** of a database ("main", "temp" or the name of an attached db). This -** function returns the index of the named database in db->aDb[], or -** -1 if the named db cannot be found. +** Deallocate a register, making available for reuse for some other +** purpose. +** +** If a register is currently being used by the column cache, then +** the dallocation is deferred until the column cache line that uses +** the register becomes stale. */ -SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ - int i = -1; /* Database number */ - if( zName ){ - Db *pDb; - int n = sqlite3Strlen30(zName); - for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && - 0==sqlite3StrICmp(pDb->zName, zName) ){ - break; +SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ + if( iReg && pParse->nTempRegaTempReg) ){ + int i; + struct yColCache *p; + for(i=0, p=pParse->aColCache; iiReg==iReg ){ + p->tempReg = 1; + return; } } + pParse->aTempReg[pParse->nTempReg++] = iReg; } - return i; } /* -** The token *pName contains the name of a database (either "main" or -** "temp" or the name of an attached db). This routine returns the -** index of the named database in db->aDb[], or -1 if the named db -** does not exist. +** Allocate or deallocate a block of nReg consecutive registers */ -SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){ - int i; /* Database number */ - char *zName; /* Name we are searching for */ - zName = sqlite3NameFromToken(db, pName); - i = sqlite3FindDbName(db, zName); - sqlite3DbFree(db, zName); +SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ + int i, n; + i = pParse->iRangeReg; + n = pParse->nRangeReg; + if( nReg<=n ){ + assert( !usedAsColumnCache(pParse, i, i+n-1) ); + pParse->iRangeReg += nReg; + pParse->nRangeReg -= nReg; + }else{ + i = pParse->nMem+1; + pParse->nMem += nReg; + } return i; } +SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + sqlite3ExprCacheRemove(pParse, iReg, nReg); + if( nReg>pParse->nRangeReg ){ + pParse->nRangeReg = nReg; + pParse->iRangeReg = iReg; + } +} -/* The table or view or trigger name is passed to this routine via tokens -** pName1 and pName2. If the table name was fully qualified, for example: -** -** CREATE TABLE xxx.yyy (...); -** -** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if -** the table name is not fully qualified, i.e.: +/************** End of expr.c ************************************************/ +/************** Begin file alter.c *******************************************/ +/* +** 2005 February 15 ** -** CREATE TABLE yyy(...); +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Then pName1 is set to "yyy" and pName2 is "". +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** This routine sets the *ppUnqual pointer to point at the token (pName1 or -** pName2) that stores the unqualified table name. The index of the -** database "xxx" is returned. +************************************************************************* +** This file contains C code routines that used to generate VDBE code +** that implements the ALTER TABLE command. */ -SQLITE_PRIVATE int sqlite3TwoPartName( - Parse *pParse, /* Parsing and code generating context */ - Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */ - Token *pName2, /* The "yyy" in the name "xxx.yyy" */ - Token **pUnqual /* Write the unqualified object name here */ -){ - int iDb; /* Database holding the object */ - sqlite3 *db = pParse->db; - - if( ALWAYS(pName2!=0) && pName2->n>0 ){ - if( db->init.busy ) { - sqlite3ErrorMsg(pParse, "corrupt database"); - pParse->nErr++; - return -1; - } - *pUnqual = pName2; - iDb = sqlite3FindDb(db, pName1); - if( iDb<0 ){ - sqlite3ErrorMsg(pParse, "unknown database %T", pName1); - pParse->nErr++; - return -1; - } - }else{ - assert( db->init.iDb==0 || db->init.busy ); - iDb = db->init.iDb; - *pUnqual = pName1; - } - return iDb; -} /* -** This routine is used to check if the UTF-8 string zName is a legal -** unqualified name for a new schema object (table, index, view or -** trigger). All names are legal except those that begin with the string -** "sqlite_" (in upper, lower or mixed case). This portion of the namespace -** is reserved for internal use. +** The code in this file only exists if we are not omitting the +** ALTER TABLE logic from the build. */ -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ - if( !pParse->db->init.busy && pParse->nested==0 - && (pParse->db->flags & SQLITE_WriteSchema)==0 - && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ - sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", zName); - return SQLITE_ERROR; - } - return SQLITE_OK; -} +#ifndef SQLITE_OMIT_ALTERTABLE + /* -** Begin constructing a new table representation in memory. This is -** the first of several action routines that get called in response -** to a CREATE TABLE statement. In particular, this routine is called -** after seeing tokens "CREATE" and "TABLE" and the table name. The isTemp -** flag is true if the table should be stored in the auxiliary database -** file instead of in the main database file. This is normally the case -** when the "TEMP" or "TEMPORARY" keyword occurs in between -** CREATE and TABLE. +** This function is used by SQL generated to implement the +** ALTER TABLE command. The first argument is the text of a CREATE TABLE or +** CREATE INDEX command. The second is a table name. The table name in +** the CREATE TABLE or CREATE INDEX statement is replaced with the third +** argument and the result returned. Examples: ** -** The new table record is initialized and put in pParse->pNewTable. -** As more of the CREATE TABLE statement is parsed, additional action -** routines will be called to add more information to this record. -** At the end of the CREATE TABLE statement, the sqlite3EndTable() routine -** is called to complete the construction of the new table record. +** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def') +** -> 'CREATE TABLE def(a, b, c)' +** +** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def') +** -> 'CREATE INDEX i ON def(a, b, c)' */ -SQLITE_PRIVATE void sqlite3StartTable( - Parse *pParse, /* Parser context */ - Token *pName1, /* First part of the name of the table or view */ - Token *pName2, /* Second part of the name of the table or view */ - int isTemp, /* True if this is a TEMP table */ - int isView, /* True if this is a VIEW */ - int isVirtual, /* True if this is a VIRTUAL table */ - int noErr /* Do nothing if table already exists */ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv ){ - Table *pTable; - char *zName = 0; /* The name of the new table */ - sqlite3 *db = pParse->db; - Vdbe *v; - int iDb; /* Database number to create the table in */ - Token *pName; /* Unqualified name of the table to create */ + unsigned char const *zSql = sqlite3_value_text(argv[0]); + unsigned char const *zTableName = sqlite3_value_text(argv[1]); - /* The table or view name to create is passed to this routine via tokens - ** pName1 and pName2. If the table name was fully qualified, for example: - ** - ** CREATE TABLE xxx.yyy (...); - ** - ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if - ** the table name is not fully qualified, i.e.: - ** - ** CREATE TABLE yyy(...); - ** - ** Then pName1 is set to "yyy" and pName2 is "". - ** - ** The call below sets the pName pointer to point at the token (pName1 or - ** pName2) that stores the unqualified table name. The variable iDb is - ** set to the index of the database that the table or view is to be - ** created in. - */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless - ** the database name is "temp" anyway. */ - sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); - return; - } - if( !OMIT_TEMPDB && isTemp ) iDb = 1; + int token; + Token tname; + unsigned char const *zCsr = zSql; + int len = 0; + char *zRet; - pParse->sNameToken = *pName; - zName = sqlite3NameFromToken(db, pName); - if( zName==0 ) return; - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ - goto begin_table_error; - } - if( db->init.iDb==1 ) isTemp = 1; -#ifndef SQLITE_OMIT_AUTHORIZATION - assert( (isTemp & 1)==isTemp ); - { - int code; - char *zDb = db->aDb[iDb].zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ - goto begin_table_error; - } - if( isView ){ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_VIEW; - }else{ - code = SQLITE_CREATE_VIEW; - } - }else{ - if( !OMIT_TEMPDB && isTemp ){ - code = SQLITE_CREATE_TEMP_TABLE; - }else{ - code = SQLITE_CREATE_TABLE; - } - } - if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ - goto begin_table_error; - } - } -#endif + sqlite3 *db = sqlite3_context_db_handle(context); - /* Make sure the new table name does not collide with an existing - ** index or table name in the same database. Issue an error message if - ** it does. The exception is if the statement being parsed was passed - ** to an sqlite3_declare_vtab() call. In that case only the column names - ** and types will be used, so there is no need to test for namespace - ** collisions. + UNUSED_PARAMETER(NotUsed); + + /* The principle used to locate the table name in the CREATE TABLE + ** statement is that the table name is the first non-space token that + ** is immediately followed by a TK_LP or TK_USING token. */ - if( !IN_DECLARE_VTAB ){ - char *zDb = db->aDb[iDb].zName; - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - goto begin_table_error; - } - pTable = sqlite3FindTable(db, zName, zDb); - if( pTable ){ - if( !noErr ){ - sqlite3ErrorMsg(pParse, "table %T already exists", pName); + if( zSql ){ + do { + if( !*zCsr ){ + /* Ran out of input before finding an opening bracket. Return NULL. */ + return; } - goto begin_table_error; - } - if( sqlite3FindIndex(db, zName, zDb)!=0 ){ - sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); - goto begin_table_error; - } - } - pTable = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTable==0 ){ - db->mallocFailed = 1; - pParse->rc = SQLITE_NOMEM; - pParse->nErr++; - goto begin_table_error; + /* Store the token that zCsr points to in tname. */ + tname.z = (char*)zCsr; + tname.n = len; + + /* Advance zCsr to the next token. Store that token type in 'token', + ** and its length in 'len' (to be used next iteration of this loop). + */ + do { + zCsr += len; + len = sqlite3GetToken(zCsr, &token); + } while( token==TK_SPACE ); + assert( len>0 ); + } while( token!=TK_LP && token!=TK_USING ); + + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, + zTableName, tname.z+tname.n); + sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } - pTable->zName = zName; - pTable->iPKey = -1; - pTable->pSchema = db->aDb[iDb].pSchema; - pTable->nRef = 1; - pTable->nRowEst = 1000000; - assert( pParse->pNewTable==0 ); - pParse->pNewTable = pTable; +} - /* If this is the magic sqlite_sequence table used by autoincrement, - ** then record a pointer to this table in the main database structure - ** so that INSERT can find the table easily. - */ -#ifndef SQLITE_OMIT_AUTOINCREMENT - if( !pParse->nested && strcmp(zName, "sqlite_sequence")==0 ){ - pTable->pSchema->pSeqTab = pTable; +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 1) The complete text of the CREATE TABLE statement being modified, +** 2) The old name of the table being renamed, and +** 3) The new name of the table being renamed. +** +** It returns the new CREATE TABLE statement. For example: +** +** sqlite_rename_parent('CREATE TABLE t1(a REFERENCES t2)', 't2', 't3') +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +static void renameParentFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char *zOutput = 0; + char *zResult; + unsigned char const *zInput = sqlite3_value_text(argv[0]); + unsigned char const *zOld = sqlite3_value_text(argv[1]); + unsigned char const *zNew = sqlite3_value_text(argv[2]); + + unsigned const char *z; /* Pointer to token */ + int n; /* Length of token z */ + int token; /* Type of token */ + + UNUSED_PARAMETER(NotUsed); + for(z=zInput; *z; z=z+n){ + n = sqlite3GetToken(z, &token); + if( token==TK_REFERENCES ){ + char *zParent; + do { + z += n; + n = sqlite3GetToken(z, &token); + }while( token==TK_SPACE ); + + zParent = sqlite3DbStrNDup(db, (const char *)z, n); + if( zParent==0 ) break; + sqlite3Dequote(zParent); + if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ + char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", + (zOutput?zOutput:""), z-zInput, zInput, (const char *)zNew + ); + sqlite3DbFree(db, zOutput); + zOutput = zOut; + zInput = &z[n]; + } + sqlite3DbFree(db, zParent); + } } + + zResult = sqlite3MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), + sqlite3_result_text(context, zResult, -1, SQLITE_DYNAMIC); + sqlite3DbFree(db, zOutput); +} #endif - /* Begin generating the code that will insert the table record into - ** the SQLITE_MASTER table. Note in particular that we must go ahead - ** and allocate the record number for the table entry now. Before any - ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause - ** indices to be created and the table record must come before the - ** indices. Hence, the record number for the table must be allocated - ** now. +#ifndef SQLITE_OMIT_TRIGGER +/* This function is used by SQL generated to implement the +** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER +** statement. The second is a table name. The table name in the CREATE +** TRIGGER statement is replaced with the third argument and the result +** returned. This is analagous to renameTableFunc() above, except for CREATE +** TRIGGER, not CREATE INDEX and CREATE TABLE. +*/ +static void renameTriggerFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + unsigned char const *zSql = sqlite3_value_text(argv[0]); + unsigned char const *zTableName = sqlite3_value_text(argv[1]); + + int token; + Token tname; + int dist = 3; + unsigned char const *zCsr = zSql; + int len = 0; + char *zRet; + sqlite3 *db = sqlite3_context_db_handle(context); + + UNUSED_PARAMETER(NotUsed); + + /* The principle used to locate the table name in the CREATE TRIGGER + ** statement is that the table name is the first token that is immediatedly + ** preceded by either TK_ON or TK_DOT and immediatedly followed by one + ** of TK_WHEN, TK_BEGIN or TK_FOR. */ - if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int j1; - int fileFormat; - int reg1, reg2, reg3; - sqlite3BeginWriteOperation(pParse, 0, iDb); + if( zSql ){ + do { -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( isVirtual ){ - sqlite3VdbeAddOp0(v, OP_VBegin); - } -#endif + if( !*zCsr ){ + /* Ran out of input before finding the table name. Return NULL. */ + return; + } - /* If the file format and encoding in the database have not been set, - ** set them now. - */ - reg1 = pParse->regRowid = ++pParse->nMem; - reg2 = pParse->regRoot = ++pParse->nMem; - reg3 = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); - fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? - 1 : SQLITE_MAX_FILE_FORMAT; - sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); - sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); - sqlite3VdbeJumpHere(v, j1); + /* Store the token that zCsr points to in tname. */ + tname.z = (char*)zCsr; + tname.n = len; - /* This just creates a place-holder record in the sqlite_master table. - ** The record created does not contain anything yet. It will be replaced - ** by the real entry in code generated at sqlite3EndTable(). - ** - ** The rowid for the new entry is left in register pParse->regRowid. - ** The root page number of the new table is left in reg pParse->regRoot. - ** The rowid and root page number values are needed by the code that - ** sqlite3EndTable will generate. + /* Advance zCsr to the next token. Store that token type in 'token', + ** and its length in 'len' (to be used next iteration of this loop). + */ + do { + zCsr += len; + len = sqlite3GetToken(zCsr, &token); + }while( token==TK_SPACE ); + assert( len>0 ); + + /* Variable 'dist' stores the number of tokens read since the most + ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN + ** token is read and 'dist' equals 2, the condition stated above + ** to be met. + ** + ** Note that ON cannot be a database, table or column name, so + ** there is no need to worry about syntax like + ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc. + */ + dist++; + if( token==TK_DOT || token==TK_ON ){ + dist = 0; + } + } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) ); + + /* Variable tname now contains the token that is the old table-name + ** in the CREATE TRIGGER statement. */ -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) - if( isView || isVirtual ){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); - }else -#endif - { - sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); - } - sqlite3OpenMasterTable(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); - sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3VdbeAddOp0(v, OP_Close); + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, + zTableName, tname.z+tname.n); + sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } +} +#endif /* !SQLITE_OMIT_TRIGGER */ - /* Normal (non-error) return. */ - return; +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +SQLITE_PRIVATE void sqlite3AlterFunctions(void){ + static SQLITE_WSD FuncDef aAlterTableFuncs[] = { + FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), +#ifndef SQLITE_OMIT_TRIGGER + FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), +#endif + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); - /* If an error occurs, we jump here */ -begin_table_error: - sqlite3DbFree(db, zName); - return; + for(i=0; i OR name= OR ... +** +** If argument zWhere is NULL, then a pointer string containing the text +** "name=" is returned, where is the quoted version +** of the string passed as argument zConstant. The returned buffer is +** allocated using sqlite3DbMalloc(). It is the responsibility of the +** caller to ensure that it is eventually freed. +** +** If argument zWhere is not NULL, then the string returned is +** " OR name=", where is the contents of zWhere. +** In this case zWhere is passed to sqlite3DbFree() before returning. +** */ -#define STRICMP(x, y) (\ -sqlite3UpperToLower[*(unsigned char *)(x)]== \ -sqlite3UpperToLower[*(unsigned char *)(y)] \ -&& sqlite3StrICmp((x)+1,(y)+1)==0 ) +static char *whereOrName(sqlite3 *db, char *zWhere, char *zConstant){ + char *zNew; + if( !zWhere ){ + zNew = sqlite3MPrintf(db, "name=%Q", zConstant); + }else{ + zNew = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, zConstant); + sqlite3DbFree(db, zWhere); + } + return zNew; +} +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) /* -** Add a new column to the table currently being constructed. -** -** The parser calls this routine once for each column declaration -** in a CREATE TABLE statement. sqlite3StartTable() gets called -** first to get things going. Then this routine is called for each -** column. +** Generate the text of a WHERE expression which can be used to select all +** tables that have foreign key constraints that refer to table pTab (i.e. +** constraints for which pTab is the parent table) from the sqlite_master +** table. */ -SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ - Table *p; - int i; - char *z; - Column *pCol; - sqlite3 *db = pParse->db; - if( (p = pParse->pNewTable)==0 ) return; -#if SQLITE_MAX_COLUMN - if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ - sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); - return; +static char *whereForeignKeys(Parse *pParse, Table *pTab){ + FKey *p; + char *zWhere = 0; + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + zWhere = whereOrName(pParse->db, zWhere, p->pFrom->zName); } + return zWhere; +} #endif - z = sqlite3NameFromToken(db, pName); - if( z==0 ) return; - for(i=0; inCol; i++){ - if( STRICMP(z, p->aCol[i].zName) ){ - sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); - sqlite3DbFree(db, z); - return; + +/* +** Generate the text of a WHERE expression which can be used to select all +** temporary triggers on table pTab from the sqlite_temp_master table. If +** table pTab has no temporary triggers, or is itself stored in the +** temporary database, NULL is returned. +*/ +static char *whereTempTriggers(Parse *pParse, Table *pTab){ + Trigger *pTrig; + char *zWhere = 0; + const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */ + + /* If the table is not located in the temp-db (in which case NULL is + ** returned, loop through the tables list of triggers. For each trigger + ** that is not part of the temp-db schema, add a clause to the WHERE + ** expression being built up in zWhere. + */ + if( pTab->pSchema!=pTempSchema ){ + sqlite3 *db = pParse->db; + for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ + if( pTrig->pSchema==pTempSchema ){ + zWhere = whereOrName(db, zWhere, pTrig->zName); + } } } - if( (p->nCol & 0x7)==0 ){ - Column *aNew; - aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0])); - if( aNew==0 ){ - sqlite3DbFree(db, z); - return; - } - p->aCol = aNew; + if( zWhere ){ + char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); + sqlite3DbFree(pParse->db, zWhere); + zWhere = zNew; } - pCol = &p->aCol[p->nCol]; - memset(pCol, 0, sizeof(p->aCol[0])); - pCol->zName = z; - - /* If there is no type specified, columns have the default affinity - ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will - ** be called next to set pCol->affinity correctly. - */ - pCol->affinity = SQLITE_AFF_NONE; - p->nCol++; + return zWhere; } /* -** This routine is called by the parser while in the middle of -** parsing a CREATE TABLE statement. A "NOT NULL" constraint has -** been seen on a column. This routine sets the notNull flag on -** the column currently under construction. +** Generate code to drop and reload the internal representation of table +** pTab from the database, including triggers and temporary triggers. +** Argument zName is the name of the table in the database schema at +** the time the generated code is executed. This can be different from +** pTab->zName if this function is being called to code part of an +** "ALTER TABLE RENAME TO" statement. */ -SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ - Table *p; - p = pParse->pNewTable; - if( p==0 || NEVER(p->nCol<1) ) return; - p->aCol[p->nCol-1].notNull = (u8)onError; -} +static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ + Vdbe *v; + char *zWhere; + int iDb; /* Index of database containing pTab */ +#ifndef SQLITE_OMIT_TRIGGER + Trigger *pTrig; +#endif -/* -** Scan the column type name zType (length nType) and return the -** associated affinity type. -** -** This routine does a case-independent search of zType for the -** substrings in the following table. If one of the substrings is -** found, the corresponding affinity is returned. If zType contains -** more than one of the substrings, entries toward the top of -** the table take priority. For example, if zType is 'BLOBINT', -** SQLITE_AFF_INTEGER is returned. -** -** Substring | Affinity -** -------------------------------- -** 'INT' | SQLITE_AFF_INTEGER -** 'CHAR' | SQLITE_AFF_TEXT -** 'CLOB' | SQLITE_AFF_TEXT -** 'TEXT' | SQLITE_AFF_TEXT -** 'BLOB' | SQLITE_AFF_NONE -** 'REAL' | SQLITE_AFF_REAL -** 'FLOA' | SQLITE_AFF_REAL -** 'DOUB' | SQLITE_AFF_REAL -** -** If none of the substrings in the above table are found, -** SQLITE_AFF_NUMERIC is returned. -*/ -SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn){ - u32 h = 0; - char aff = SQLITE_AFF_NUMERIC; + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 ); - if( zIn ) while( zIn[0] ){ - h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; - zIn++; - if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ - aff = SQLITE_AFF_TEXT; - }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ - aff = SQLITE_AFF_TEXT; - }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ - aff = SQLITE_AFF_TEXT; - }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ - && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ - aff = SQLITE_AFF_NONE; -#ifndef SQLITE_OMIT_FLOATING_POINT - }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ - && aff==SQLITE_AFF_NUMERIC ){ - aff = SQLITE_AFF_REAL; - }else if( h==(('f'<<24)+('l'<<16)+('o'<<8)+'a') /* FLOA */ - && aff==SQLITE_AFF_NUMERIC ){ - aff = SQLITE_AFF_REAL; - }else if( h==(('d'<<24)+('o'<<16)+('u'<<8)+'b') /* DOUB */ - && aff==SQLITE_AFF_NUMERIC ){ - aff = SQLITE_AFF_REAL; -#endif - }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){ /* INT */ - aff = SQLITE_AFF_INTEGER; - break; - } +#ifndef SQLITE_OMIT_TRIGGER + /* Drop any table triggers from the internal schema. */ + for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ + int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + assert( iTrigDb==iDb || iTrigDb==1 ); + sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->zName, 0); } +#endif - return aff; -} + /* Drop the table and index from the internal schema. */ + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); -/* -** This routine is called by the parser while in the middle of -** parsing a CREATE TABLE statement. The pFirst token is the first -** token in the sequence of tokens that describe the type of the -** column currently under construction. pLast is the last token -** in the sequence. Use this information to construct a string -** that contains the typename of the column and store that string -** in zType. -*/ -SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ - Table *p; - Column *pCol; + /* Reload the table, index and permanent trigger schemas. */ + zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); + if( !zWhere ) return; + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); - p = pParse->pNewTable; - if( p==0 || NEVER(p->nCol<1) ) return; - pCol = &p->aCol[p->nCol-1]; - assert( pCol->zType==0 ); - pCol->zType = sqlite3NameFromToken(pParse->db, pType); - pCol->affinity = sqlite3AffinityType(pCol->zType); +#ifndef SQLITE_OMIT_TRIGGER + /* Now, if the table is not stored in the temp database, reload any temp + ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. + */ + if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); + } +#endif } /* -** The expression is the default value for the most recently added column -** of the table currently under construction. -** -** Default value expressions must be constant. Raise an exception if this -** is not the case. +** Parameter zName is the name of a table that is about to be altered +** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). +** If the table is a system table, this function leaves an error message +** in pParse->zErr (system tables may not be altered) and returns non-zero. ** -** This routine is called by the parser while in the middle of -** parsing a CREATE TABLE statement. +** Or, if zName is not a system table, zero is returned. */ -SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ - Table *p; - Column *pCol; - sqlite3 *db = pParse->db; - p = pParse->pNewTable; - if( p!=0 ){ - pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ - sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", - pCol->zName); - }else{ - /* A copy of pExpr is used instead of the original, as pExpr contains - ** tokens that point to volatile memory. The 'span' of the expression - ** is required by pragma table_info. - */ - sqlite3ExprDelete(db, pCol->pDflt); - pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); - sqlite3DbFree(db, pCol->zDflt); - pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); - } +static int isSystemTable(Parse *pParse, const char *zName){ + if( sqlite3Strlen30(zName)>6 && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ + sqlite3ErrorMsg(pParse, "table %s may not be altered", zName); + return 1; } - sqlite3ExprDelete(db, pSpan->pExpr); + return 0; } /* -** Designate the PRIMARY KEY for the table. pList is a list of names -** of columns that form the primary key. If pList is NULL, then the -** most recently added column of the table is the primary key. -** -** A table can have at most one primary key. If the table already has -** a primary key (and this is the second primary key) then create an -** error. -** -** If the PRIMARY KEY is on a single column whose datatype is INTEGER, -** then we will try to use that column as the rowid. Set the Table.iPKey -** field of the table under construction to be the index of the -** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is -** no INTEGER PRIMARY KEY. -** -** If the key is not an INTEGER PRIMARY KEY, then create a unique -** index for the key. No index is created for INTEGER PRIMARY KEYs. +** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" +** command. */ -SQLITE_PRIVATE void sqlite3AddPrimaryKey( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* List of field names to be indexed */ - int onError, /* What to do with a uniqueness conflict */ - int autoInc, /* True if the AUTOINCREMENT keyword is present */ - int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ +SQLITE_PRIVATE void sqlite3AlterRenameTable( + Parse *pParse, /* Parser context. */ + SrcList *pSrc, /* The table to rename. */ + Token *pName /* The new table name. */ ){ - Table *pTab = pParse->pNewTable; - char *zType = 0; - int iCol = -1, i; - if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; - if( pTab->tabFlags & TF_HasPrimaryKey ){ + int iDb; /* Database that contains the table */ + char *zDb; /* Name of database iDb */ + Table *pTab; /* Table being renamed */ + char *zName = 0; /* NULL-terminated version of pName */ + sqlite3 *db = pParse->db; /* Database connection */ + int nTabName; /* Number of UTF-8 characters in zTabName */ + const char *zTabName; /* Original name of the table */ + Vdbe *v; +#ifndef SQLITE_OMIT_TRIGGER + char *zWhere = 0; /* Where clause to locate temp triggers */ +#endif + VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ + int savedDbFlags; /* Saved value of db->flags */ + + savedDbFlags = db->flags; + if( NEVER(db->mallocFailed) ) goto exit_rename_table; + assert( pSrc->nSrc==1 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + + pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + if( !pTab ) goto exit_rename_table; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + zDb = db->aDb[iDb].zName; + db->flags |= SQLITE_PreferBuiltin; + + /* Get a NULL terminated version of the new table name. */ + zName = sqlite3NameFromToken(db, pName); + if( !zName ) goto exit_rename_table; + + /* Check that a table or index named 'zName' does not already exist + ** in database iDb. If so, this is an error. + */ + if( sqlite3FindTable(db, zName, zDb) || sqlite3FindIndex(db, zName, zDb) ){ sqlite3ErrorMsg(pParse, - "table \"%s\" has more than one primary key", pTab->zName); - goto primary_key_exit; + "there is already another table or index with this name: %s", zName); + goto exit_rename_table; } - pTab->tabFlags |= TF_HasPrimaryKey; - if( pList==0 ){ - iCol = pTab->nCol - 1; - pTab->aCol[iCol].isPrimKey = 1; - }else{ - for(i=0; inExpr; i++){ - for(iCol=0; iColnCol; iCol++){ - if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ - break; - } - } - if( iColnCol ){ - pTab->aCol[iCol].isPrimKey = 1; - } - } - if( pList->nExpr>1 ) iCol = -1; + + /* Make sure it is not a system table being altered, or a reserved name + ** that the table is being renamed to. + */ + if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + goto exit_rename_table; } - if( iCol>=0 && iColnCol ){ - zType = pTab->aCol[iCol].zType; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto + exit_rename_table; } - if( zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC ){ - pTab->iPKey = iCol; - pTab->keyConf = (u8)onError; - assert( autoInc==0 || autoInc==1 ); - pTab->tabFlags |= autoInc*TF_Autoincrement; - }else if( autoInc ){ -#ifndef SQLITE_OMIT_AUTOINCREMENT - sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " - "INTEGER PRIMARY KEY"); -#endif - }else{ - Index *p; - p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0); - if( p ){ - p->autoIndex = 2; - } - pList = 0; + +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); + goto exit_rename_table; } +#endif -primary_key_exit: - sqlite3ExprListDelete(pParse->db, pList); - return; -} +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_table; + } +#endif -/* -** Add a new CHECK constraint to the table currently under construction. -*/ -SQLITE_PRIVATE void sqlite3AddCheckConstraint( - Parse *pParse, /* Parsing context */ - Expr *pCheckExpr /* The check expression */ -){ - sqlite3 *db = pParse->db; -#ifndef SQLITE_OMIT_CHECK - Table *pTab = pParse->pNewTable; - if( pTab && !IN_DECLARE_VTAB ){ - pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); - }else +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_rename_table; + } + if( IsVirtual(pTab) ){ + pVTab = sqlite3GetVTable(db, pTab); + if( pVTab->pVtab->pModule->xRename==0 ){ + pVTab = 0; + } + } #endif - { - sqlite3ExprDelete(db, pCheckExpr); + + /* Begin a transaction and code the VerifyCookie for database iDb. + ** Then modify the schema cookie (since the ALTER TABLE modifies the + ** schema). Open a statement transaction if the table is a virtual + ** table. + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto exit_rename_table; } -} + sqlite3BeginWriteOperation(pParse, pVTab!=0, iDb); + sqlite3ChangeCookie(pParse, iDb); -/* -** Set the collation function of the most recently parsed table column -** to the CollSeq given. -*/ -SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ - Table *p; - int i; - char *zColl; /* Dequoted name of collation sequence */ - sqlite3 *db; + /* If this is a virtual table, invoke the xRename() function if + ** one is defined. The xRename() callback will modify the names + ** of any resources used by the v-table implementation (including other + ** SQLite tables) that are identified by the name of the virtual table. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pVTab ){ + int i = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); + sqlite3MayAbort(pParse); + } +#endif - if( (p = pParse->pNewTable)==0 ) return; - i = p->nCol-1; - db = pParse->db; - zColl = sqlite3NameFromToken(db, pToken); - if( !zColl ) return; + /* figure out how many UTF-8 characters are in zName */ + zTabName = pTab->zName; + nTabName = sqlite3Utf8CharLen(zTabName, -1); - if( sqlite3LocateCollSeq(pParse, zColl) ){ - Index *pIdx; - p->aCol[i].zColl = zColl; - - /* If the column is declared as " PRIMARY KEY COLLATE ", - ** then an index may have been created on this column before the - ** collation type was added. Correct this if it is the case. - */ - for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ - assert( pIdx->nColumn==1 ); - if( pIdx->aiColumn[0]==i ){ - pIdx->azColl[0] = p->aCol[i].zColl; - } +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + if( db->flags&SQLITE_ForeignKeys ){ + /* If foreign-key support is enabled, rewrite the CREATE TABLE + ** statements corresponding to all child tables of foreign key constraints + ** for which the renamed table is the parent table. */ + if( (zWhere=whereForeignKeys(pParse, pTab))!=0 ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = sqlite_rename_parent(sql, %Q, %Q) " + "WHERE %s;", zDb, SCHEMA_TABLE(iDb), zTabName, zName, zWhere); + sqlite3DbFree(db, zWhere); } - }else{ - sqlite3DbFree(db, zColl); } -} +#endif -/* -** This function returns the collation sequence for database native text -** encoding identified by the string zName, length nName. -** -** If the requested collation sequence is not available, or not available -** in the database native encoding, the collation factory is invoked to -** request it. If the collation factory does not supply such a sequence, -** and the sequence is available in another text encoding, then that is -** returned instead. -** -** If no versions of the requested collations sequence are available, or -** another error occurs, NULL is returned and an error message written into -** pParse. -** -** This routine is a wrapper around sqlite3FindCollSeq(). This routine -** invokes the collation factory if the named collation cannot be found -** and generates an error message. -** -** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() -*/ -SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ - sqlite3 *db = pParse->db; - u8 enc = ENC(db); - u8 initbusy = db->init.busy; - CollSeq *pColl; + /* Modify the sqlite_master table to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s SET " +#ifdef SQLITE_OMIT_TRIGGER + "sql = sqlite_rename_table(sql, %Q), " +#else + "sql = CASE " + "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)" + "ELSE sqlite_rename_table(sql, %Q) END, " +#endif + "tbl_name = %Q, " + "name = CASE " + "WHEN type='table' THEN %Q " + "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " + "ELSE name END " + "WHERE tbl_name=%Q AND " + "(type='table' OR type='index' OR type='trigger');", + zDb, SCHEMA_TABLE(iDb), zName, zName, zName, +#ifndef SQLITE_OMIT_TRIGGER + zName, +#endif + zName, nTabName, zTabName + ); - pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); - if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(db, enc, pColl, zName); - if( !pColl ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* If the sqlite_sequence table exists in this database, then update + ** it with the new table name. + */ + if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", + zDb, zName, pTab->zName); + } +#endif + +#ifndef SQLITE_OMIT_TRIGGER + /* If there are TEMP triggers on this table, modify the sqlite_temp_master + ** table. Don't do this if the table being ALTERed is itself located in + ** the temp database. + */ + if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ + sqlite3NestedParse(pParse, + "UPDATE sqlite_temp_master SET " + "sql = sqlite_rename_trigger(sql, %Q), " + "tbl_name = %Q " + "WHERE %s;", zName, zName, zWhere); + sqlite3DbFree(db, zWhere); + } +#endif + +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + if( db->flags&SQLITE_ForeignKeys ){ + FKey *p; + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + Table *pFrom = p->pFrom; + if( pFrom!=pTab ){ + reloadTableSchema(pParse, p->pFrom, pFrom->zName); + } } } +#endif - return pColl; + /* Drop and reload the internal table schema. */ + reloadTableSchema(pParse, pTab, zName); + +exit_rename_table: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zName); + db->flags = savedDbFlags; } /* -** Generate code that will increment the schema cookie. -** -** The schema cookie is used to determine when the schema for the -** database changes. After each schema change, the cookie value -** changes. When a process first reads the schema it records the -** cookie. Thereafter, whenever it goes to access the database, -** it checks the cookie to make sure the schema has not changed -** since it was last read. -** -** This plan is not completely bullet-proof. It is possible for -** the schema to change multiple times and for the cookie to be -** set back to prior value. But schema changes are infrequent -** and the probability of hitting the same cookie value is only -** 1 chance in 2^32. So we're safe enough. +** Generate code to make sure the file format number is at least minFormat. +** The generated code will increase the file format number if necessary. */ -SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3 *db = pParse->db; - Vdbe *v = pParse->pVdbe; - sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, r1); - sqlite3ReleaseTempReg(pParse, r1); +SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ + Vdbe *v; + v = sqlite3GetVdbe(pParse); + /* The VDBE should have been allocated before this routine is called. + ** If that allocation failed, we would have quit before reaching this + ** point */ + if( ALWAYS(v) ){ + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + int j1; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); + j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); + sqlite3VdbeJumpHere(v, j1); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } } /* -** Measure the number of characters needed to output the given -** identifier. The number returned includes any quotes used -** but does not include the null terminator. +** This function is called after an "ALTER TABLE ... ADD" statement +** has been parsed. Argument pColDef contains the text of the new +** column definition. ** -** The estimate is conservative. It might be larger that what is -** really needed. +** The Table structure pParse->pNewTable was extended to include +** the new column during parsing. */ -static int identLength(const char *z){ - int n; - for(n=0; *z; n++, z++){ - if( *z=='"' ){ n++; } - } - return n + 2; -} +SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ + Table *pNew; /* Copy of pParse->pNewTable */ + Table *pTab; /* Table being altered */ + int iDb; /* Database number */ + const char *zDb; /* Database name */ + const char *zTab; /* Table name */ + char *zCol; /* Null-terminated column definition */ + Column *pCol; /* The new column */ + Expr *pDflt; /* Default value for the new column */ + sqlite3 *db; /* The database connection; */ -/* -** The first parameter is a pointer to an output buffer. The second -** parameter is a pointer to an integer that contains the offset at -** which to write into the output buffer. This function copies the -** nul-terminated string pointed to by the third parameter, zSignedIdent, -** to the specified offset in the buffer and updates *pIdx to refer -** to the first byte after the last byte written before returning. -** -** If the string zSignedIdent consists entirely of alpha-numeric -** characters, does not begin with a digit and is not an SQL keyword, -** then it is copied to the output buffer exactly as it is. Otherwise, -** it is quoted using double-quotes. -*/ -static void identPut(char *z, int *pIdx, char *zSignedIdent){ - unsigned char *zIdent = (unsigned char*)zSignedIdent; - int i, j, needQuote; - i = *pIdx; + db = pParse->db; + if( pParse->nErr || db->mallocFailed ) return; + pNew = pParse->pNewTable; + assert( pNew ); - for(j=0; zIdent[j]; j++){ - if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; - } - needQuote = sqlite3Isdigit(zIdent[0]) || sqlite3KeywordCode(zIdent, j)!=TK_ID; - if( !needQuote ){ - needQuote = zIdent[j]; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pNew->pSchema); + zDb = db->aDb[iDb].zName; + zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ + pCol = &pNew->aCol[pNew->nCol-1]; + pDflt = pCol->pDflt; + pTab = sqlite3FindTable(db, zTab, zDb); + assert( pTab ); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + return; } +#endif - if( needQuote ) z[i++] = '"'; - for(j=0; zIdent[j]; j++){ - z[i++] = zIdent[j]; - if( zIdent[j]=='"' ) z[i++] = '"'; + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + if( pDflt && pDflt->op==TK_NULL ){ + pDflt = 0; } - if( needQuote ) z[i++] = '"'; - z[i] = 0; - *pIdx = i; -} -/* -** Generate a CREATE TABLE statement appropriate for the given -** table. Memory to hold the text of the statement is obtained -** from sqliteMalloc() and must be freed by the calling function. -*/ -static char *createTableStmt(sqlite3 *db, Table *p){ - int i, k, n; - char *zStmt; - char *zSep, *zSep2, *zEnd; - Column *pCol; - n = 0; - for(pCol = p->aCol, i=0; inCol; i++, pCol++){ - n += identLength(pCol->zName) + 5; + /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. + ** If there is a NOT NULL constraint, then the default value for the + ** column must not be NULL. + */ + if( pCol->isPrimKey ){ + sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); + return; } - n += identLength(p->zName); - if( n<50 ){ - zSep = ""; - zSep2 = ","; - zEnd = ")"; - }else{ - zSep = "\n "; - zSep2 = ",\n "; - zEnd = "\n)"; + if( pNew->pIndex ){ + sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column"); + return; } - n += 35 + 6*p->nCol; - zStmt = sqlite3DbMallocRaw(0, n); - if( zStmt==0 ){ - db->mallocFailed = 1; - return 0; + if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ + sqlite3ErrorMsg(pParse, + "Cannot add a REFERENCES column with non-NULL default value"); + return; + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorMsg(pParse, + "Cannot add a NOT NULL column with default value NULL"); + return; } - sqlite3_snprintf(n, zStmt, "CREATE TABLE "); - k = sqlite3Strlen30(zStmt); - identPut(zStmt, &k, p->zName); - zStmt[k++] = '('; - for(pCol=p->aCol, i=0; inCol; i++, pCol++){ - static const char * const azType[] = { - /* SQLITE_AFF_TEXT */ " TEXT", - /* SQLITE_AFF_NONE */ "", - /* SQLITE_AFF_NUMERIC */ " NUM", - /* SQLITE_AFF_INTEGER */ " INT", - /* SQLITE_AFF_REAL */ " REAL" - }; - int len; - const char *zType; - sqlite3_snprintf(n-k, &zStmt[k], zSep); - k += sqlite3Strlen30(&zStmt[k]); - zSep = zSep2; - identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); - assert( pCol->affinity-SQLITE_AFF_TEXT < sizeof(azType)/sizeof(azType[0]) ); - testcase( pCol->affinity==SQLITE_AFF_TEXT ); - testcase( pCol->affinity==SQLITE_AFF_NONE ); - testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); - testcase( pCol->affinity==SQLITE_AFF_INTEGER ); - testcase( pCol->affinity==SQLITE_AFF_REAL ); - - zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; - len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_NONE - || pCol->affinity==sqlite3AffinityType(zType) ); - memcpy(&zStmt[k], zType, len); - k += len; - assert( k<=n ); + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal; + if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ + db->mallocFailed = 1; + return; + } + if( !pVal ){ + sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); + return; + } + sqlite3ValueFree(pVal); } - sqlite3_snprintf(n-k, &zStmt[k], "%s", zEnd); - return zStmt; + + /* Modify the CREATE TABLE statement. */ + zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); + if( zCol ){ + char *zEnd = &zCol[pColDef->n-1]; + int savedDbFlags = db->flags; + while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ + *zEnd-- = '\0'; + } + db->flags |= SQLITE_PreferBuiltin; + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " + "WHERE type = 'table' AND name = %Q", + zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1, + zTab + ); + sqlite3DbFree(db, zCol); + db->flags = savedDbFlags; + } + + /* If the default value of the new column is NULL, then set the file + ** format to 2. If the default value of the new column is not NULL, + ** the file format becomes 3. + */ + sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2); + + /* Reload the schema of the modified table. */ + reloadTableSchema(pParse, pTab, pTab->zName); } /* -** This routine is called to report the final ")" that terminates -** a CREATE TABLE statement. -** -** The table structure that other action routines have been building -** is added to the internal hash tables, assuming no errors have -** occurred. +** This function is called by the parser after the table-name in +** an "ALTER TABLE ADD" statement is parsed. Argument +** pSrc is the full-name of the table being altered. ** -** An entry for the table is made in the master table on disk, unless -** this is a temporary table or db->init.busy==1. When db->init.busy==1 -** it means we are reading the sqlite_master table because we just -** connected to the database or because the sqlite_master table has -** recently changed, so the entry for this table already exists in -** the sqlite_master table. We do not want to create it again. +** This routine makes a (partial) copy of the Table structure +** for the table being altered and sets Parse.pNewTable to point +** to it. Routines called by the parser as the column definition +** is parsed (i.e. sqlite3AddColumn()) add the new Column data to +** the copy. The copy of the Table structure is deleted by tokenize.c +** after parsing is finished. ** -** If the pSelect argument is not NULL, it means that this routine -** was called to create a table generated from a -** "CREATE TABLE ... AS SELECT ..." statement. The column names of -** the new table will match the result set of the SELECT. +** Routine sqlite3AlterFinishAddColumn() will be called to complete +** coding the "ALTER TABLE ... ADD" statement. */ -SQLITE_PRIVATE void sqlite3EndTable( - Parse *pParse, /* Parse context */ - Token *pCons, /* The ',' token after the last column defn. */ - Token *pEnd, /* The final ')' token in the CREATE TABLE */ - Select *pSelect /* Select from a "CREATE ... AS SELECT" */ -){ - Table *p; - sqlite3 *db = pParse->db; +SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ + Table *pNew; + Table *pTab; + Vdbe *v; int iDb; + int i; + int nAlloc; + sqlite3 *db = pParse->db; - if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ - return; + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( db->mallocFailed ) goto exit_begin_add_column; + pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); + if( !pTab ) goto exit_begin_add_column; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be altered"); + goto exit_begin_add_column; } - p = pParse->pNewTable; - if( p==0 ) return; +#endif - assert( !db->init.busy || !pSelect ); + /* Make sure this is not an attempt to ALTER a view. */ + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); + goto exit_begin_add_column; + } + if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + goto exit_begin_add_column; + } - iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( pTab->addColOffset>0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); -#ifndef SQLITE_OMIT_CHECK - /* Resolve names in all CHECK constraint expressions. + /* Put a copy of the Table struct in Parse.pNewTable for the + ** sqlite3AddColumn() function and friends to modify. But modify + ** the name by adding an "sqlite_altertab_" prefix. By adding this + ** prefix, we insure that the name will not collide with an existing + ** table because user table are not allowed to have the "sqlite_" + ** prefix on their name. */ - if( p->pCheck ){ - SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ - NameContext sNC; /* Name context for pParse->pNewTable */ - - memset(&sNC, 0, sizeof(sNC)); - memset(&sSrc, 0, sizeof(sSrc)); - sSrc.nSrc = 1; - sSrc.a[0].zName = p->zName; - sSrc.a[0].pTab = p; - sSrc.a[0].iCursor = -1; - sNC.pParse = pParse; - sNC.pSrcList = &sSrc; - sNC.isCheck = 1; - if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ - return; - } + pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); + if( !pNew ) goto exit_begin_add_column; + pParse->pNewTable = pNew; + pNew->nRef = 1; + pNew->nCol = pTab->nCol; + assert( pNew->nCol>0 ); + nAlloc = (((pNew->nCol-1)/8)*8)+8; + assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); + if( !pNew->aCol || !pNew->zName ){ + db->mallocFailed = 1; + goto exit_begin_add_column; } -#endif /* !defined(SQLITE_OMIT_CHECK) */ - - /* If the db->init.busy is 1 it means we are reading the SQL off the - ** "sqlite_master" or "sqlite_temp_master" table on the disk. - ** So do not write to the disk again. Extract the root page number - ** for the table from the db->init.newTnum field. (The page number - ** should have been put there by the sqliteOpenCb routine.) - */ - if( db->init.busy ){ - p->tnum = db->init.newTnum; + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + for(i=0; inCol; i++){ + Column *pCol = &pNew->aCol[i]; + pCol->zName = sqlite3DbStrDup(db, pCol->zName); + pCol->zColl = 0; + pCol->zType = 0; + pCol->pDflt = 0; + pCol->zDflt = 0; } + pNew->pSchema = db->aDb[iDb].pSchema; + pNew->addColOffset = pTab->addColOffset; + pNew->nRef = 1; - /* If not initializing, then create a record for the new table - ** in the SQLITE_MASTER table of the database. - ** - ** If this is a TEMPORARY table, write the entry into the auxiliary - ** file instead of into the main database file. - */ - if( !db->init.busy ){ - int n; - Vdbe *v; - char *zType; /* "view" or "table" */ - char *zType2; /* "VIEW" or "TABLE" */ - char *zStmt; /* Text of the CREATE TABLE or CREATE VIEW statement */ + /* Begin a transaction and increment the schema cookie. */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + v = sqlite3GetVdbe(pParse); + if( !v ) goto exit_begin_add_column; + sqlite3ChangeCookie(pParse, iDb); - v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; +exit_begin_add_column: + sqlite3SrcListDelete(db, pSrc); + return; +} +#endif /* SQLITE_ALTER_TABLE */ - sqlite3VdbeAddOp1(v, OP_Close, 0); +/************** End of alter.c ***********************************************/ +/************** Begin file analyze.c *****************************************/ +/* +** 2005 July 8 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code associated with the ANALYZE command. +*/ +#ifndef SQLITE_OMIT_ANALYZE - /* - ** Initialize zType for the new view or table. - */ - if( p->pSelect==0 ){ - /* A regular table */ - zType = "table"; - zType2 = "TABLE"; -#ifndef SQLITE_OMIT_VIEW - }else{ - /* A view */ - zType = "view"; - zType2 = "VIEW"; +/* +** This routine generates code that opens the sqlite_stat1 table for +** writing with cursor iStatCur. If the library was built with the +** SQLITE_ENABLE_STAT2 macro defined, then the sqlite_stat2 table is +** opened for writing using cursor (iStatCur+1) +** +** If the sqlite_stat1 tables does not previously exist, it is created. +** Similarly, if the sqlite_stat2 table does not exist and the library +** is compiled with SQLITE_ENABLE_STAT2 defined, it is created. +** +** Argument zWhere may be a pointer to a buffer containing a table name, +** or it may be a NULL pointer. If it is not NULL, then all entries in +** the sqlite_stat1 and (if applicable) sqlite_stat2 tables associated +** with the named table are deleted. If zWhere==0, then code is generated +** to delete all stat table entries. +*/ +static void openStatTable( + Parse *pParse, /* Parsing context */ + int iDb, /* The database we are looking in */ + int iStatCur, /* Open the sqlite_stat1 table on this cursor */ + const char *zWhere, /* Delete entries for this table or index */ + const char *zWhereType /* Either "tbl" or "idx" */ +){ + static const struct { + const char *zName; + const char *zCols; + } aTable[] = { + { "sqlite_stat1", "tbl,idx,stat" }, +#ifdef SQLITE_ENABLE_STAT2 + { "sqlite_stat2", "tbl,idx,sampleno,sample" }, #endif - } + }; - /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT - ** statement to populate the new table. The root-page number for the - ** new table is in register pParse->regRoot. - ** - ** Once the SELECT has been coded by sqlite3Select(), it is in a - ** suitable state to query for the column names and types to be used - ** by the new table. - ** - ** A shared-cache write-lock is not required to write to the new table, - ** as a schema-lock must have already been obtained to create it. Since - ** a schema-lock excludes all other database users, the write-lock would - ** be redundant. - */ - if( pSelect ){ - SelectDest dest; - Table *pSelTab; + int aRoot[] = {0, 0}; + u8 aCreateTbl[] = {0, 0}; - assert(pParse->nTab==1); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); - sqlite3VdbeChangeP5(v, 1); - pParse->nTab = 2; - sqlite3SelectDestInit(&dest, SRT_Table, 1); - sqlite3Select(pParse, pSelect, &dest); - sqlite3VdbeAddOp1(v, OP_Close, 1); - if( pParse->nErr==0 ){ - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); - if( pSelTab==0 ) return; - assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; - p->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - } - } + int i; + sqlite3 *db = pParse->db; + Db *pDb; + Vdbe *v = sqlite3GetVdbe(pParse); + if( v==0 ) return; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3VdbeDb(v)==db ); + pDb = &db->aDb[iDb]; - /* Compute the complete text of the CREATE statement */ - if( pSelect ){ - zStmt = createTableStmt(db, p); - }else{ - n = (int)(pEnd->z - pParse->sNameToken.z) + 1; - zStmt = sqlite3MPrintf(db, - "CREATE %s %.*s", zType2, n, pParse->sNameToken.z + for(i=0; izName))==0 ){ + /* The sqlite_stat[12] table does not exist. Create it. Note that a + ** side-effect of the CREATE TABLE statement is to leave the rootpage + ** of the new table in register pParse->regRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols ); - } - - /* A slot for the record has already been allocated in the - ** SQLITE_MASTER table. We just need to update that slot with all - ** the information we've collected. - */ - sqlite3NestedParse(pParse, - "UPDATE %Q.%s " - "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " - "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - zType, - p->zName, - p->zName, - pParse->regRoot, - zStmt, - pParse->regRowid - ); - sqlite3DbFree(db, zStmt); - sqlite3ChangeCookie(pParse, iDb); - -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* Check to see if we need to create an sqlite_sequence table for - ** keeping track of autoincrement keys. - */ - if( p->tabFlags & TF_Autoincrement ){ - Db *pDb = &db->aDb[iDb]; - if( pDb->pSchema->pSeqTab==0 ){ + aRoot[i] = pParse->regRoot; + aCreateTbl[i] = 1; + }else{ + /* The table already exists. If zWhere is not NULL, delete all entries + ** associated with the table zWhere. If zWhere is NULL, delete the + ** entire contents of the table. */ + aRoot[i] = pStat->tnum; + sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); + if( zWhere ){ sqlite3NestedParse(pParse, - "CREATE TABLE %Q.sqlite_sequence(name,seq)", - pDb->zName + "DELETE FROM %Q.%s WHERE %s=%Q", pDb->zName, zTab, zWhereType, zWhere ); + }else{ + /* The sqlite_stat[12] table already exists. Delete all rows. */ + sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); } } -#endif - - /* Reparse everything to update our internal data structures */ - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); } - - /* Add the table to the in-memory representation of the database. - */ - if( db->init.busy ){ - Table *pOld; - Schema *pSchema = p->pSchema; - pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, - sqlite3Strlen30(p->zName),p); - if( pOld ){ - assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ - db->mallocFailed = 1; - return; - } - pParse->pNewTable = 0; - db->nTable++; - db->flags |= SQLITE_InternChanges; - -#ifndef SQLITE_OMIT_ALTERTABLE - if( !p->pSelect ){ - const char *zName = (const char *)pParse->sNameToken.z; - int nName; - assert( !pSelect && pCons && pEnd ); - if( pCons->z==0 ){ - pCons = pEnd; - } - nName = (int)((const char *)pCons->z - zName); - p->addColOffset = 13 + sqlite3Utf8CharLen(zName, nName); - } -#endif + /* Open the sqlite_stat[12] tables for writing. */ + for(i=0; idb; + sqlite3 *db = pParse->db; /* Database handle */ + Index *pIdx; /* An index to being analyzed */ + int iIdxCur; /* Cursor open on index being analyzed */ + Vdbe *v; /* The virtual machine being built up */ + int i; /* Loop counter */ + int topOfLoop; /* The top of the loop */ + int endOfLoop; /* The end of the loop */ + int jZeroRows = -1; /* Jump from here if number of rows is zero */ + int iDb; /* Index of database containing pTab */ + int regTabname = iMem++; /* Register containing table name */ + int regIdxname = iMem++; /* Register containing index name */ + int regSampleno = iMem++; /* Register containing next sample number */ + int regCol = iMem++; /* Content of a column analyzed table */ + int regRec = iMem++; /* Register holding completed record */ + int regTemp = iMem++; /* Temporary use register */ + int regRowid = iMem++; /* Rowid for the inserted record */ - if( pParse->nVar>0 ){ - sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(db, pSelect); +#ifdef SQLITE_ENABLE_STAT2 + int addr = 0; /* Instruction address */ + int regTemp2 = iMem++; /* Temporary use register */ + int regSamplerecno = iMem++; /* Index of next sample to record */ + int regRecno = iMem++; /* Current sample index */ + int regLast = iMem++; /* Index of last sample to record */ + int regFirst = iMem++; /* Index of first sample to record */ +#endif + + v = sqlite3GetVdbe(pParse); + if( v==0 || NEVER(pTab==0) ){ return; } - sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); - p = pParse->pNewTable; - if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(db, pSelect); + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ return; } - sqlite3TwoPartName(pParse, pName1, pName2, &pName); - iDb = sqlite3SchemaToIndex(db, p->pSchema); - if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) - && sqlite3FixSelect(&sFix, pSelect) - ){ - sqlite3SelectDelete(db, pSelect); + if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + /* Do not gather statistics on system tables */ return; } - - /* Make a copy of the entire SELECT statement that defines the view. - ** This will force all the Expr.token.z values to be dynamically - ** allocated rather than point to the input string - which means that - ** they will persist after the current sqlite3_exec() call returns. - */ - p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed ){ + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, + db->aDb[iDb].zName ) ){ return; } - if( !db->init.busy ){ - sqlite3ViewGetColumnNames(pParse, p); - } +#endif - /* Locate the end of the CREATE VIEW statement. Make sEnd point to - ** the end. - */ - sEnd = pParse->sLastToken; - if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ - sEnd.z += sEnd.n; - } - sEnd.n = 0; - n = (int)(sEnd.z - pBegin->z); - z = pBegin->z; - while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } - sEnd.z = &z[n-1]; - sEnd.n = 1; + /* Establish a read-lock on the table at the shared-cache level. */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ - sqlite3EndTable(pParse, 0, &sEnd, 0); - return; -} -#endif /* SQLITE_OMIT_VIEW */ + iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int nCol; + KeyInfo *pKey; -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) -/* -** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. -*/ -SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ - Table *pSelTab; /* A fake table from which we get the result set */ - Select *pSel; /* Copy of the SELECT that implements the view */ - int nErr = 0; /* Number of errors encountered */ - int n; /* Temporarily holds the number of cursors assigned */ - sqlite3 *db = pParse->db; /* Database connection for malloc errors */ - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; + nCol = pIdx->nColumn; + pKey = sqlite3IndexKeyinfo(pParse, pIdx); + if( iMem+1+(nCol*2)>pParse->nMem ){ + pParse->nMem = iMem+1+(nCol*2); + } - assert( pTable ); + /* Open a cursor to the index to be analyzed. */ + assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); + sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, + (char *)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3VtabCallConnect(pParse, pTable) ){ - return SQLITE_ERROR; - } - if( IsVirtual(pTable) ) return 0; + /* Populate the register containing the index name. */ + sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); + +#ifdef SQLITE_ENABLE_STAT2 + + /* If this iteration of the loop is generating code to analyze the + ** first index in the pTab->pIndex list, then register regLast has + ** not been populated. In this case populate it now. */ + if( pTab->pIndex==pIdx ){ + sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regSamplerecno); + sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2-1, regTemp); + sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2, regTemp2); + + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regLast); + sqlite3VdbeAddOp2(v, OP_Null, 0, regFirst); + addr = sqlite3VdbeAddOp3(v, OP_Lt, regSamplerecno, 0, regLast); + sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regFirst); + sqlite3VdbeAddOp3(v, OP_Multiply, regLast, regTemp, regLast); + sqlite3VdbeAddOp2(v, OP_AddImm, regLast, SQLITE_INDEX_SAMPLES*2-2); + sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regLast); + sqlite3VdbeJumpHere(v, addr); + } + + /* Zero the regSampleno and regRecno registers. */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regSampleno); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRecno); + sqlite3VdbeAddOp2(v, OP_Copy, regFirst, regSamplerecno); #endif -#ifndef SQLITE_OMIT_VIEW - /* A positive nCol means the columns names for this view are - ** already known. - */ - if( pTable->nCol>0 ) return 0; + /* The block of memory cells initialized here is used as follows. + ** + ** iMem: + ** The total number of rows in the table. + ** + ** iMem+1 .. iMem+nCol: + ** Number of distinct entries in index considering the + ** left-most N columns only, where N is between 1 and nCol, + ** inclusive. + ** + ** iMem+nCol+1 .. Mem+2*nCol: + ** Previous value of indexed columns, from left to right. + ** + ** Cells iMem through iMem+nCol are initialized to 0. The others are + ** initialized to contain an SQL NULL. + */ + for(i=0; i<=nCol; i++){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem+i); + } + for(i=0; inCol<0 ){ - sqlite3ErrorMsg(pParse, "view %s is circularly defined", pTable->zName); - return 1; - } - assert( pTable->nCol>=0 ); + /* Start the analysis loop. This loop runs through all the entries in + ** the index b-tree. */ + endOfLoop = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop); + topOfLoop = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1); - /* If we get this far, it means we need to compute the table names. - ** Note that the call to sqlite3ResultSetOfSelect() will expand any - ** "*" elements in the results set of the view and will assign cursors - ** to the elements of the FROM clause. But we do not want these changes - ** to be permanent. So the computation is done on a copy of the SELECT - ** statement that defines the view. - */ - assert( pTable->pSelect ); - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); - if( pSel ){ - u8 enableLookaside = db->lookaside.bEnabled; - n = pParse->nTab; - sqlite3SrcListAssignCursors(pParse, pSel->pSrc); - pTable->nCol = -1; - db->lookaside.bEnabled = 0; -#ifndef SQLITE_OMIT_AUTHORIZATION - xAuth = db->xAuth; - db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); - db->xAuth = xAuth; -#else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + for(i=0; ilookaside.bEnabled = enableLookaside; - pParse->nTab = n; - if( pSelTab ){ - assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; - pTable->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - pTable->pSchema->flags |= DB_UnresetViews; - }else{ - pTable->nCol = 0; - nErr++; + + /* Always record the very first row */ + sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1); + } + assert( pIdx->azColl!=0 ); + assert( pIdx->azColl[i]!=0 ); + pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); + sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1, + (char*)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); } - sqlite3SelectDelete(db, pSel); - } else { - nErr++; + if( db->mallocFailed ){ + /* If a malloc failure has occurred, then the result of the expression + ** passed as the second argument to the call to sqlite3VdbeJumpHere() + ** below may be negative. Which causes an assert() to fail (or an + ** out-of-bounds write if SQLITE_DEBUG is not defined). */ + return; + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); + for(i=0; i0 then it is always the case the D>0 so division by zero + ** is never possible. + */ + sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); + if( jZeroRows<0 ){ + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); + } + for(i=0; ipIndex==0 ){ + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); + VdbeComment((v, "%s", pTab->zName)); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); + sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regSampleno); + }else{ + sqlite3VdbeJumpHere(v, jZeroRows); + jZeroRows = sqlite3VdbeAddOp0(v, OP_Goto); + } + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + if( pParse->nMemnMem = regRec; + sqlite3VdbeJumpHere(v, jZeroRows); } -#endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifndef SQLITE_OMIT_VIEW /* -** Clear the column names from every VIEW in database idx. +** Generate code that will cause the most recent index analysis to +** be loaded into internal hash tables where is can be used. */ -static void sqliteViewResetAll(sqlite3 *db, int idx){ - HashElem *i; - if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; - for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ - Table *pTab = sqliteHashData(i); - if( pTab->pSelect ){ - sqliteDeleteColumnNames(db, pTab); - pTab->aCol = 0; - pTab->nCol = 0; - } +static void loadAnalysis(Parse *pParse, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); } - DbClearProperty(db, idx, DB_UnresetViews); } -#else -# define sqliteViewResetAll(A,B) -#endif /* SQLITE_OMIT_VIEW */ /* -** This function is called by the VDBE to adjust the internal schema -** used by SQLite when the btree layer moves a table root page. The -** root-page of a table or index in database iDb has changed from iFrom -** to iTo. -** -** Ticket #1728: The symbol table might still contain information -** on tables and/or indices that are the process of being deleted. -** If you are unlucky, one of those deleted indices or tables might -** have the same rootpage number as the real table or index that is -** being moved. So we cannot stop searching after the first match -** because the first match might be for one of the deleted indices -** or tables and not the table/index that is actually being moved. -** We must continue looping until all tables and indices with -** rootpage==iFrom have been converted to have a rootpage of iTo -** in order to be certain that we got the right one. +** Generate code that will do an analysis of an entire database */ -#ifndef SQLITE_OMIT_AUTOVACUUM -SQLITE_PRIVATE void sqlite3RootPageMoved(Db *pDb, int iFrom, int iTo){ - HashElem *pElem; - Hash *pHash; +static void analyzeDatabase(Parse *pParse, int iDb){ + sqlite3 *db = pParse->db; + Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + HashElem *k; + int iStatCur; + int iMem; - pHash = &pDb->pSchema->tblHash; - for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ - Table *pTab = sqliteHashData(pElem); - if( pTab->tnum==iFrom ){ - pTab->tnum = iTo; - } - } - pHash = &pDb->pSchema->idxHash; - for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ - Index *pIdx = sqliteHashData(pElem); - if( pIdx->tnum==iFrom ){ - pIdx->tnum = iTo; - } + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 2; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem); } + loadAnalysis(pParse, iDb); } -#endif /* -** Write code to erase the table with root-page iTable from database iDb. -** Also write code to modify the sqlite_master table and internal schema -** if a root-page of another table is moved by the btree-layer whilst -** erasing iTable (this can happen with an auto-vacuum database). -*/ -static void destroyRootPage(Parse *pParse, int iTable, int iDb){ - Vdbe *v = sqlite3GetVdbe(pParse); - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); - sqlite3MayAbort(pParse); -#ifndef SQLITE_OMIT_AUTOVACUUM - /* OP_Destroy stores an in integer r1. If this integer - ** is non-zero, then it is the root page number of a table moved to - ** location iTable. The following code modifies the sqlite_master table to - ** reflect this. - ** - ** The "#NNN" in the SQL is a special constant that means whatever value - ** is in register NNN. See grammar rules associated with the TK_REGISTER - ** token for additional information. - */ - sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", - pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); -#endif - sqlite3ReleaseTempReg(pParse, r1); +** Generate code that will do an analysis of a single table in +** a database. If pOnlyIdx is not NULL then it is a single index +** in pTab that should be analyzed. +*/ +static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ + int iDb; + int iStatCur; + + assert( pTab!=0 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 2; + if( pOnlyIdx ){ + openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); + }else{ + openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); + } + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1); + loadAnalysis(pParse, iDb); } /* -** Write VDBE code to erase table pTab and all associated indices on disk. -** Code to update the sqlite_master tables and internal schema definitions -** in case a root-page belonging to another table is moved by the btree layer -** is also added (this can happen with an auto-vacuum database). +** Generate code for the ANALYZE command. The parser calls this routine +** when it recognizes an ANALYZE command. +** +** ANALYZE -- 1 +** ANALYZE -- 2 +** ANALYZE ?.? -- 3 +** +** Form 1 causes all indices in all attached databases to be analyzed. +** Form 2 analyzes all indices the single database named. +** Form 3 analyzes all indices associated with the named table. */ -static void destroyTable(Parse *pParse, Table *pTab){ -#ifdef SQLITE_OMIT_AUTOVACUUM +SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ + sqlite3 *db = pParse->db; + int iDb; + int i; + char *z, *zDb; + Table *pTab; Index *pIdx; - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - destroyRootPage(pParse, pTab->tnum, iDb); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - destroyRootPage(pParse, pIdx->tnum, iDb); - } -#else - /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM - ** is not defined), then it is important to call OP_Destroy on the - ** table and index root-pages in order, starting with the numerically - ** largest root-page number. This guarantees that none of the root-pages - ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the - ** following were coded: - ** - ** OP_Destroy 4 0 - ** ... - ** OP_Destroy 5 0 - ** - ** and root page 5 happened to be the largest root-page number in the - ** database, then root page 5 would be moved to page 4 by the - ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit - ** a free-list page. - */ - int iTab = pTab->tnum; - int iDestroyed = 0; + Token *pTableName; - while( 1 ){ - Index *pIdx; - int iLargest = 0; + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } - if( iDestroyed==0 || iTabpIndex; pIdx; pIdx=pIdx->pNext){ - int iIdx = pIdx->tnum; - assert( pIdx->pSchema==pTab->pSchema ); - if( (iDestroyed==0 || (iIdxiLargest ){ - iLargest = iIdx; - } + assert( pName2!=0 || pName1==0 ); + if( pName1==0 ){ + /* Form 1: Analyze everything */ + for(i=0; inDb; i++){ + if( i==1 ) continue; /* Do not analyze the TEMP database */ + analyzeDatabase(pParse, i); } - if( iLargest==0 ){ - return; + }else if( pName2->n==0 ){ + /* Form 2: Analyze the database or table named */ + iDb = sqlite3FindDb(db, pName1); + if( iDb>=0 ){ + analyzeDatabase(pParse, iDb); }else{ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - destroyRootPage(pParse, iLargest, iDb); - iDestroyed = iLargest; + z = sqlite3NameFromToken(db, pName1); + if( z ){ + if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ + analyzeTable(pParse, pIdx->pTable, pIdx); + }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ + analyzeTable(pParse, pTab, 0); + } + sqlite3DbFree(db, z); + } } + }else{ + /* Form 3: Analyze the fully qualified table name */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); + if( iDb>=0 ){ + zDb = db->aDb[iDb].zName; + z = sqlite3NameFromToken(db, pTableName); + if( z ){ + if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ + analyzeTable(pParse, pIdx->pTable, pIdx); + }else if( (pTab = sqlite3LocateTable(pParse, 0, z, zDb))!=0 ){ + analyzeTable(pParse, pTab, 0); + } + sqlite3DbFree(db, z); + } + } } -#endif } /* -** This routine is called to do the work of a DROP TABLE statement. -** pName is the name of the table to be dropped. +** Used to pass information from the analyzer reader through to the +** callback routine. */ -SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ - Table *pTab; - Vdbe *v; - sqlite3 *db = pParse->db; - int iDb; - - if( db->mallocFailed ){ - goto exit_drop_table; - } - assert( pParse->nErr==0 ); - assert( pName->nSrc==1 ); - if( noErr ) db->suppressErr++; - pTab = sqlite3LocateTable(pParse, isView, - pName->a[0].zName, pName->a[0].zDatabase); - if( noErr ) db->suppressErr--; - - if( pTab==0 ){ - goto exit_drop_table; - } - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( iDb>=0 && iDbnDb ); +typedef struct analysisInfo analysisInfo; +struct analysisInfo { + sqlite3 *db; + const char *zDatabase; +}; - /* If pTab is a virtual table, call ViewGetColumnNames() to ensure - ** it is initialized. - */ - if( IsVirtual(pTab) && sqlite3ViewGetColumnNames(pParse, pTab) ){ - goto exit_drop_table; +/* +** This callback is invoked once for each index when reading the +** sqlite_stat1 table. +** +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. +*/ +static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ + analysisInfo *pInfo = (analysisInfo*)pData; + Index *pIndex; + Table *pTable; + int i, c, n; + unsigned int v; + const char *z; + + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + + if( argv==0 || argv[0]==0 || argv[2]==0 ){ + return 0; } -#ifndef SQLITE_OMIT_AUTHORIZATION - { - int code; - const char *zTab = SCHEMA_TABLE(iDb); - const char *zDb = db->aDb[iDb].zName; - const char *zArg2 = 0; - if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ - goto exit_drop_table; - } - if( isView ){ - if( !OMIT_TEMPDB && iDb==1 ){ - code = SQLITE_DROP_TEMP_VIEW; - }else{ - code = SQLITE_DROP_VIEW; - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( IsVirtual(pTab) ){ - code = SQLITE_DROP_VTABLE; - zArg2 = sqlite3GetVTable(db, pTab)->pMod->zName; -#endif - }else{ - if( !OMIT_TEMPDB && iDb==1 ){ - code = SQLITE_DROP_TEMP_TABLE; - }else{ - code = SQLITE_DROP_TABLE; - } + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ + return 0; + } + if( argv[1] ){ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + }else{ + pIndex = 0; + } + n = pIndex ? pIndex->nColumn : 0; + z = argv[2]; + for(i=0; *z && i<=n; i++){ + v = 0; + while( (c=z[0])>='0' && c<='9' ){ + v = v*10 + c - '0'; + z++; } - if( sqlite3AuthCheck(pParse, code, pTab->zName, zArg2, zDb) ){ - goto exit_drop_table; + if( i==0 ) pTable->nRowEst = v; + if( pIndex==0 ) break; + pIndex->aiRowEst[i] = v; + if( *z==' ' ) z++; + if( memcmp(z, "unordered", 10)==0 ){ + pIndex->bUnordered = 1; + break; } - if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){ - goto exit_drop_table; + } + return 0; +} + +/* +** If the Index.aSample variable is not NULL, delete the aSample[] array +** and its contents. +*/ +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ +#ifdef SQLITE_ENABLE_STAT2 + if( pIdx->aSample ){ + int j; + for(j=0; jaSample[j]; + if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ + sqlite3DbFree(db, p->u.z); + } } + sqlite3DbFree(db, pIdx->aSample); } +#else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pIdx); #endif - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ - sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); - goto exit_drop_table; +} + +/* +** Load the content of the sqlite_stat1 and sqlite_stat2 tables. The +** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] +** arrays. The contents of sqlite_stat2 are used to populate the +** Index.aSample[] arrays. +** +** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR +** is returned. In this case, even if SQLITE_ENABLE_STAT2 was defined +** during compilation and the sqlite_stat2 table is present, no data is +** read from it. +** +** If SQLITE_ENABLE_STAT2 was defined during compilation and the +** sqlite_stat2 table is not present in the database, SQLITE_ERROR is +** returned. However, in this case, data is read from the sqlite_stat1 +** table (if it is present) before returning. +** +** If an OOM error occurs, this function always sets db->mallocFailed. +** This means if the caller does not care about other errors, the return +** code may be ignored. +*/ +SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ + analysisInfo sInfo; + HashElem *i; + char *zSql; + int rc; + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + + /* Clear any prior statistics */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3DefaultRowEst(pIdx); + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; } -#ifndef SQLITE_OMIT_VIEW - /* Ensure DROP TABLE is not used on a view, and DROP VIEW is not used - ** on a table. - */ - if( isView && pTab->pSelect==0 ){ - sqlite3ErrorMsg(pParse, "use DROP TABLE to delete table %s", pTab->zName); - goto exit_drop_table; + /* Check to make sure the sqlite_stat1 table exists */ + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ + return SQLITE_ERROR; } - if( !isView && pTab->pSelect ){ - sqlite3ErrorMsg(pParse, "use DROP VIEW to delete view %s", pTab->zName); - goto exit_drop_table; + + /* Load new statistics out of the sqlite_stat1 table */ + zSql = sqlite3MPrintf(db, + "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); } -#endif - /* Generate code to remove the table from the master table - ** on disk. - */ - v = sqlite3GetVdbe(pParse); - if( v ){ - Trigger *pTrigger; - Db *pDb = &db->aDb[iDb]; - sqlite3BeginWriteOperation(pParse, 1, iDb); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp0(v, OP_VBegin); - } -#endif - sqlite3FkDropTable(pParse, pName, pTab); + /* Load the statistics from the sqlite_stat2 table. */ +#ifdef SQLITE_ENABLE_STAT2 + if( rc==SQLITE_OK && !sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase) ){ + rc = SQLITE_ERROR; + } + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; - /* Drop all triggers associated with the table being dropped. Code - ** is generated to remove entries from sqlite_master and/or - ** sqlite_temp_master if required. - */ - pTrigger = sqlite3TriggerList(pParse, pTab); - while( pTrigger ){ - assert( pTrigger->pSchema==pTab->pSchema || - pTrigger->pSchema==db->aDb[1].pSchema ); - sqlite3DropTriggerPtr(pParse, pTrigger); - pTrigger = pTrigger->pNext; + zSql = sqlite3MPrintf(db, + "SELECT idx,sampleno,sample FROM %Q.sqlite_stat2", sInfo.zDatabase); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); } -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* Remove any entries of the sqlite_sequence table associated with - ** the table being dropped. This is done before the table is dropped - ** at the btree level, in case the sqlite_sequence table needs to - ** move as a result of the drop (can happen in auto-vacuum mode). - */ - if( pTab->tabFlags & TF_Autoincrement ){ - sqlite3NestedParse(pParse, - "DELETE FROM %s.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName - ); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + pIdx = zIndex ? sqlite3FindIndex(db, zIndex, sInfo.zDatabase) : 0; + if( pIdx ){ + int iSample = sqlite3_column_int(pStmt, 1); + if( iSample=0 ){ + int eType = sqlite3_column_type(pStmt, 2); + + if( pIdx->aSample==0 ){ + static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; + pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); + if( pIdx->aSample==0 ){ + db->mallocFailed = 1; + break; + } + memset(pIdx->aSample, 0, sz); + } + + assert( pIdx->aSample ); + { + IndexSample *pSample = &pIdx->aSample[iSample]; + pSample->eType = (u8)eType; + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + pSample->u.r = sqlite3_column_double(pStmt, 2); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const char *z = (const char *)( + (eType==SQLITE_BLOB) ? + sqlite3_column_blob(pStmt, 2): + sqlite3_column_text(pStmt, 2) + ); + int n = sqlite3_column_bytes(pStmt, 2); + if( n>24 ){ + n = 24; + } + pSample->nByte = (u8)n; + if( n < 1){ + pSample->u.z = 0; + }else{ + pSample->u.z = sqlite3DbStrNDup(0, z, n); + if( pSample->u.z==0 ){ + db->mallocFailed = 1; + break; + } + } + } + } + } + } + } + rc = sqlite3_finalize(pStmt); } + } #endif - /* Drop all SQLITE_MASTER table and index entries that refer to the - ** table. The program name loops through the master table and deletes - ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be - ** created in the temp database that refers to a table in another - ** database. - */ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + } + return rc; +} - /* Drop any statistics from the sqlite_stat1 table, if it exists */ - if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q", pDb->zName, pTab->zName - ); - } - if( !isView && !IsVirtual(pTab) ){ - destroyTable(pParse, pTab); - } +#endif /* SQLITE_OMIT_ANALYZE */ - /* Remove the table entry from SQLite's internal schema and modify - ** the schema cookie. - */ - if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); +/************** End of analyze.c *********************************************/ +/************** Begin file attach.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the ATTACH and DETACH commands. +*/ + +#ifndef SQLITE_OMIT_ATTACH +/* +** Resolve an expression that was part of an ATTACH or DETACH statement. This +** is slightly different from resolving a normal SQL expression, because simple +** identifiers are treated as strings, not possible column names or aliases. +** +** i.e. if the parser sees: +** +** ATTACH DATABASE abc AS def +** +** it treats the two expressions as literal strings 'abc' and 'def' instead of +** looking for columns of the same name. +** +** This only applies to the root node of pExpr, so the statement: +** +** ATTACH DATABASE abc||def AS 'db2' +** +** will fail because neither abc or def can be resolved. +*/ +static int resolveAttachExpr(NameContext *pName, Expr *pExpr) +{ + int rc = SQLITE_OK; + if( pExpr ){ + if( pExpr->op!=TK_ID ){ + rc = sqlite3ResolveExprNames(pName, pExpr); + if( rc==SQLITE_OK && !sqlite3ExprIsConstant(pExpr) ){ + sqlite3ErrorMsg(pName->pParse, "invalid name: \"%s\"", pExpr->u.zToken); + return SQLITE_ERROR; + } + }else{ + pExpr->op = TK_STRING; } - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - sqlite3ChangeCookie(pParse, iDb); } - sqliteViewResetAll(db, iDb); - -exit_drop_table: - sqlite3SrcListDelete(db, pName); + return rc; } /* -** This routine is called to create a new foreign key on the table -** currently under construction. pFromCol determines which columns -** in the current table point to the foreign key. If pFromCol==0 then -** connect the key to the last column inserted. pTo is the name of -** the table referred to. pToCol is a list of tables in the other -** pTo table that the foreign key points to. flags contains all -** information about the conflict resolution algorithms specified -** in the ON DELETE, ON UPDATE and ON INSERT clauses. +** An SQL user-function registered to do the work of an ATTACH statement. The +** three arguments to the function come directly from an attach statement: ** -** An FKey structure is created and added to the table currently -** under construction in the pParse->pNewTable field. +** ATTACH DATABASE x AS y KEY z ** -** The foreign key is set for IMMEDIATE processing. A subsequent call -** to sqlite3DeferForeignKey() might change this to DEFERRED. +** SELECT sqlite_attach(x, y, z) +** +** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the +** third argument. */ -SQLITE_PRIVATE void sqlite3CreateForeignKey( - Parse *pParse, /* Parsing context */ - ExprList *pFromCol, /* Columns in this table that point to other table */ - Token *pTo, /* Name of the other table */ - ExprList *pToCol, /* Columns in the other table */ - int flags /* Conflict resolution algorithms. */ +static void attachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv ){ - sqlite3 *db = pParse->db; -#ifndef SQLITE_OMIT_FOREIGN_KEY - FKey *pFKey = 0; - FKey *pNextTo; - Table *p = pParse->pNewTable; - int nByte; int i; - int nCol; - char *z; + int rc = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zName; + const char *zFile; + char *zPath = 0; + char *zErr = 0; + unsigned int flags; + Db *aNew; + char *zErrDyn = 0; + sqlite3_vfs *pVfs; - assert( pTo!=0 ); - if( p==0 || IN_DECLARE_VTAB ) goto fk_end; - if( pFromCol==0 ){ - int iCol = p->nCol-1; - if( NEVER(iCol<0) ) goto fk_end; - if( pToCol && pToCol->nExpr!=1 ){ - sqlite3ErrorMsg(pParse, "foreign key on %s" - " should reference only one column of table %T", - p->aCol[iCol].zName, pTo); - goto fk_end; - } - nCol = 1; - }else if( pToCol && pToCol->nExpr!=pFromCol->nExpr ){ - sqlite3ErrorMsg(pParse, - "number of columns in foreign key does not match the number of " - "columns in the referenced table"); - goto fk_end; - }else{ - nCol = pFromCol->nExpr; - } - nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; - if( pToCol ){ - for(i=0; inExpr; i++){ - nByte += sqlite3Strlen30(pToCol->a[i].zName) + 1; - } - } - pFKey = sqlite3DbMallocZero(db, nByte ); - if( pFKey==0 ){ - goto fk_end; + UNUSED_PARAMETER(NotUsed); + + zFile = (const char *)sqlite3_value_text(argv[0]); + zName = (const char *)sqlite3_value_text(argv[1]); + if( zFile==0 ) zFile = ""; + if( zName==0 ) zName = ""; + + /* Check for the following errors: + ** + ** * Too many attached databases, + ** * Transaction currently open + ** * Specified database name already being used. + */ + if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ + zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", + db->aLimit[SQLITE_LIMIT_ATTACHED] + ); + goto attach_error; } - pFKey->pFrom = p; - pFKey->pNextFrom = p->pFKey; - z = (char*)&pFKey->aCol[nCol]; - pFKey->zTo = z; - memcpy(z, pTo->z, pTo->n); - z[pTo->n] = 0; - sqlite3Dequote(z); - z += pTo->n+1; - pFKey->nCol = nCol; - if( pFromCol==0 ){ - pFKey->aCol[0].iFrom = p->nCol-1; - }else{ - for(i=0; inCol; j++){ - if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ - pFKey->aCol[i].iFrom = j; - break; - } - } - if( j>=p->nCol ){ - sqlite3ErrorMsg(pParse, - "unknown column \"%s\" in foreign key definition", - pFromCol->a[i].zName); - goto fk_end; - } - } + if( !db->autoCommit ){ + zErrDyn = sqlite3MPrintf(db, "cannot ATTACH database within transaction"); + goto attach_error; } - if( pToCol ){ - for(i=0; ia[i].zName); - pFKey->aCol[i].zCol = z; - memcpy(z, pToCol->a[i].zName, n); - z[n] = 0; - z += n+1; + for(i=0; inDb; i++){ + char *z = db->aDb[i].zName; + assert( z && zName ); + if( sqlite3StrICmp(z, zName)==0 ){ + zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); + goto attach_error; } } - pFKey->isDeferred = 0; - pFKey->aAction[0] = (u8)(flags & 0xff); /* ON DELETE action */ - pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ - pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, - pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey - ); - if( pNextTo==pFKey ){ - db->mallocFailed = 1; - goto fk_end; - } - if( pNextTo ){ - assert( pNextTo->pPrevTo==0 ); - pFKey->pNextTo = pNextTo; - pNextTo->pPrevTo = pFKey; + /* Allocate the new entry in the db->aDb[] array and initialise the schema + ** hash tables. + */ + if( db->aDb==db->aDbStatic ){ + aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); + if( aNew==0 ) return; + memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); + }else{ + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + if( aNew==0 ) return; } + db->aDb = aNew; + aNew = &db->aDb[db->nDb]; + memset(aNew, 0, sizeof(*aNew)); - /* Link the foreign key to the table as the last step. + /* Open the database file. If the btree is successfully opened, use + ** it to obtain the database schema. At this point the schema may + ** or may not be initialised. */ - p->pFKey = pFKey; - pFKey = 0; - -fk_end: - sqlite3DbFree(db, pFKey); -#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ - sqlite3ExprListDelete(db, pFromCol); - sqlite3ExprListDelete(db, pToCol); -} + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); + sqlite3_free( zPath ); + db->nDb++; + if( rc==SQLITE_CONSTRAINT ){ + rc = SQLITE_ERROR; + zErrDyn = sqlite3MPrintf(db, "database is already attached"); + }else if( rc==SQLITE_OK ){ + Pager *pPager; + aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); + if( !aNew->pSchema ){ + rc = SQLITE_NOMEM; + }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ + zErrDyn = sqlite3MPrintf(db, + "attached databases must use the same text encoding as main database"); + rc = SQLITE_ERROR; + } + pPager = sqlite3BtreePager(aNew->pBt); + sqlite3PagerLockingMode(pPager, db->dfltLockMode); + sqlite3BtreeSecureDelete(aNew->pBt, + sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); + } + aNew->safety_level = 3; + aNew->zName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zName==0 ){ + rc = SQLITE_NOMEM; + } -/* -** This routine is called when an INITIALLY IMMEDIATE or INITIALLY DEFERRED -** clause is seen as part of a foreign key definition. The isDeferred -** parameter is 1 for INITIALLY DEFERRED and 0 for INITIALLY IMMEDIATE. -** The behavior of the most recently created foreign key is adjusted -** accordingly. -*/ -SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ -#ifndef SQLITE_OMIT_FOREIGN_KEY - Table *pTab; - FKey *pFKey; - if( (pTab = pParse->pNewTable)==0 || (pFKey = pTab->pFKey)==0 ) return; - assert( isDeferred==0 || isDeferred==1 ); /* EV: R-30323-21917 */ - pFKey->isDeferred = (u8)isDeferred; -#endif -} -/* -** Generate code that will erase and refill index *pIdx. This is -** used to initialize a newly created index or to recompute the -** content of an index in response to a REINDEX command. -** -** if memRootPage is not negative, it means that the index is newly -** created. The register specified by memRootPage contains the -** root page number of the index. If memRootPage is negative, then -** the index already exists and must be cleared before being refilled and -** the root page number of the index is taken from pIndex->tnum. -*/ -static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ - Table *pTab = pIndex->pTable; /* The table that is indexed */ - int iTab = pParse->nTab++; /* Btree cursor used for pTab */ - int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ - int addr1; /* Address of top of loop */ - int tnum; /* Root page of index */ - Vdbe *v; /* Generate code into this virtual machine */ - KeyInfo *pKey; /* KeyInfo for index */ - int regIdxKey; /* Registers containing the index key */ - int regRecord; /* Register holding assemblied index record */ - sqlite3 *db = pParse->db; /* The database connection */ - int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); +#ifdef SQLITE_HAS_CODEC + if( rc==SQLITE_OK ){ + extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); + extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + int nKey; + char *zKey; + int t = sqlite3_value_type(argv[2]); + switch( t ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); + rc = SQLITE_ERROR; + break; + + case SQLITE_TEXT: + case SQLITE_BLOB: + nKey = sqlite3_value_bytes(argv[2]); + zKey = (char *)sqlite3_value_blob(argv[2]); + rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + break; -#ifndef SQLITE_OMIT_AUTHORIZATION - if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - db->aDb[iDb].zName ) ){ - return; + case SQLITE_NULL: + /* No key specified. Use the key from the main database */ + sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); + if( nKey>0 || sqlite3BtreeGetReserve(db->aDb[0].pBt)>0 ){ + rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); + } + break; + } } #endif - /* Require a write-lock on the table to perform this operation */ - sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); - - v = sqlite3GetVdbe(pParse); - if( v==0 ) return; - if( memRootPage>=0 ){ - tnum = memRootPage; - }else{ - tnum = pIndex->tnum; - sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); + /* If the file was opened successfully, read the schema for the new database. + ** If this fails, or if opening the file failed, then close the file and + ** remove the entry from the db->aDb[] array. i.e. put everything back the way + ** we found it. + */ + if( rc==SQLITE_OK ){ + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrDyn); + sqlite3BtreeLeaveAll(db); } - pKey = sqlite3IndexKeyinfo(pParse, pIndex); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, - (char *)pKey, P4_KEYINFO_HANDOFF); - if( memRootPage>=0 ){ - sqlite3VdbeChangeP5(v, 1); + if( rc ){ + int iDb = db->nDb - 1; + assert( iDb>=2 ); + if( db->aDb[iDb].pBt ){ + sqlite3BtreeClose(db->aDb[iDb].pBt); + db->aDb[iDb].pBt = 0; + db->aDb[iDb].pSchema = 0; + } + sqlite3ResetInternalSchema(db, -1); + db->nDb = iDb; + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; + sqlite3DbFree(db, zErrDyn); + zErrDyn = sqlite3MPrintf(db, "out of memory"); + }else if( zErrDyn==0 ){ + zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + } + goto attach_error; } - sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); - regRecord = sqlite3GetTempReg(pParse); - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); + + return; - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); +attach_error: + /* Return an error if we get here */ + if( zErrDyn ){ + sqlite3_result_error(context, zErrDyn, -1); + sqlite3DbFree(db, zErrDyn); } - sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp1(v, OP_Close, iTab); - sqlite3VdbeAddOp1(v, OP_Close, iIdx); + if( rc ) sqlite3_result_error_code(context, rc); } /* -** Create a new index for an SQL table. pName1.pName2 is the name of the index -** and pTblList is the name of the table that is to be indexed. Both will -** be NULL for a primary key or an index that is created to satisfy a -** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable -** as the table to be indexed. pParse->pNewTable is a table that is -** currently being constructed by a CREATE TABLE statement. +** An SQL user-function registered to do the work of an DETACH statement. The +** three arguments to the function come directly from a detach statement: ** -** pList is a list of columns to be indexed. pList will be NULL if this -** is a primary key or unique-constraint on the most recent column added -** to the table currently under construction. +** DETACH DATABASE x ** -** If the index is created successfully, return a pointer to the new Index -** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.autoIndex==2). +** SELECT sqlite_detach(x) */ -SQLITE_PRIVATE Index *sqlite3CreateIndex( - Parse *pParse, /* All information about this parse */ - Token *pName1, /* First part of index name. May be NULL */ - Token *pName2, /* Second part of index name. May be NULL */ - SrcList *pTblName, /* Table to index. Use pParse->pNewTable if 0 */ - ExprList *pList, /* A list of columns to be indexed */ - int onError, /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - Token *pStart, /* The CREATE token that begins this statement */ - Token *pEnd, /* The ")" that closes the CREATE INDEX statement */ - int sortOrder, /* Sort order of primary key when pList==NULL */ - int ifNotExist /* Omit error if index already exists */ +static void detachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv ){ - Index *pRet = 0; /* Pointer to return */ - Table *pTab = 0; /* Table to be indexed */ - Index *pIndex = 0; /* The index to be created */ - char *zName = 0; /* Name of the index */ - int nName; /* Number of characters in zName */ - int i, j; - Token nullId; /* Fake token for an empty ID list */ - DbFixer sFix; /* For assigning database names to pTable */ - int sortOrderMask; /* 1 to honor DESC in index. 0 to ignore. */ - sqlite3 *db = pParse->db; - Db *pDb; /* The specific table containing the indexed database */ - int iDb; /* Index of the database that is being written */ - Token *pName = 0; /* Unqualified name of the index to create */ - struct ExprList_item *pListItem; /* For looping over pList */ - int nCol; - int nExtra = 0; - char *zExtra; - - assert( pStart==0 || pEnd!=0 ); /* pEnd must be non-NULL if pStart is */ - assert( pParse->nErr==0 ); /* Never called with prior errors */ - if( db->mallocFailed || IN_DECLARE_VTAB ){ - goto exit_create_index; - } - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - goto exit_create_index; - } - - /* - ** Find the table that is to be indexed. Return early if not found. - */ - if( pTblName!=0 ){ - - /* Use the two-part index name to determine the database - ** to search for the table. 'Fix' the table name to this db - ** before looking up the table. - */ - assert( pName1 && pName2 ); - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ) goto exit_create_index; + const char *zName = (const char *)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + int i; + Db *pDb = 0; + char zErr[128]; -#ifndef SQLITE_OMIT_TEMPDB - /* If the index name was unqualified, check if the the table - ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. - */ - if( !db->init.busy ){ - pTab = sqlite3SrcListLookup(pParse, pTblName); - if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ - iDb = 1; - } - } -#endif + UNUSED_PARAMETER(NotUsed); - if( sqlite3FixInit(&sFix, pParse, iDb, "index", pName) && - sqlite3FixSrcList(&sFix, pTblName) - ){ - /* Because the parser constructs pTblName from a single identifier, - ** sqlite3FixSrcList can never fail. */ - assert(0); - } - pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, - pTblName->a[0].zDatabase); - if( !pTab || db->mallocFailed ) goto exit_create_index; - assert( db->aDb[iDb].pSchema==pTab->pSchema ); - }else{ - assert( pName==0 ); - pTab = pParse->pNewTable; - if( !pTab ) goto exit_create_index; - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + if( zName==0 ) zName = ""; + for(i=0; inDb; i++){ + pDb = &db->aDb[i]; + if( pDb->pBt==0 ) continue; + if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; } - pDb = &db->aDb[iDb]; - assert( pTab!=0 ); - assert( pParse->nErr==0 ); - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ - sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); - goto exit_create_index; + if( i>=db->nDb ){ + sqlite3_snprintf(sizeof(zErr),zErr, "no such database: %s", zName); + goto detach_error; } -#ifndef SQLITE_OMIT_VIEW - if( pTab->pSelect ){ - sqlite3ErrorMsg(pParse, "views may not be indexed"); - goto exit_create_index; + if( i<2 ){ + sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); + goto detach_error; } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - sqlite3ErrorMsg(pParse, "virtual tables may not be indexed"); - goto exit_create_index; + if( !db->autoCommit ){ + sqlite3_snprintf(sizeof(zErr), zErr, + "cannot DETACH database within transaction"); + goto detach_error; + } + if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){ + sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); + goto detach_error; } -#endif - /* - ** Find the name of the index. Make sure there is not already another - ** index or table with the same name. - ** - ** Exception: If we are reading the names of permanent indices from the - ** sqlite_master table (because some other process changed the schema) and - ** one of the index names collides with the name of a temporary table or - ** index, then we will continue to process this index. - ** - ** If pName==0 it means that we are - ** dealing with a primary key or UNIQUE constraint. We have to invent our - ** own name. - */ - if( pName ){ - zName = sqlite3NameFromToken(db, pName); - if( zName==0 ) goto exit_create_index; - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ - goto exit_create_index; - } - if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ - sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); - goto exit_create_index; - } - } - if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ - if( !ifNotExist ){ - sqlite3ErrorMsg(pParse, "index %s already exists", zName); - } - goto exit_create_index; - } - }else{ - int n; - Index *pLoop; - for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} - zName = sqlite3MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); - if( zName==0 ){ - goto exit_create_index; - } + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + sqlite3ResetInternalSchema(db, -1); + return; + +detach_error: + sqlite3_result_error(context, zErr, -1); +} + +/* +** This procedure generates VDBE code for a single invocation of either the +** sqlite_detach() or sqlite_attach() SQL user functions. +*/ +static void codeAttach( + Parse *pParse, /* The parser context */ + int type, /* Either SQLITE_ATTACH or SQLITE_DETACH */ + FuncDef const *pFunc,/* FuncDef wrapper for detachFunc() or attachFunc() */ + Expr *pAuthArg, /* Expression to pass to authorization callback */ + Expr *pFilename, /* Name of database file */ + Expr *pDbname, /* Name of the database to use internally */ + Expr *pKey /* Database key for encryption extension */ +){ + int rc; + NameContext sName; + Vdbe *v; + sqlite3* db = pParse->db; + int regArgs; + + memset(&sName, 0, sizeof(NameContext)); + sName.pParse = pParse; + + if( + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) + ){ + pParse->nErr++; + goto attach_end; } - /* Check for authorization to create an index. - */ #ifndef SQLITE_OMIT_AUTHORIZATION - { - const char *zDb = pDb->zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ - goto exit_create_index; + if( pAuthArg ){ + char *zAuthArg; + if( pAuthArg->op==TK_STRING ){ + zAuthArg = pAuthArg->u.zToken; + }else{ + zAuthArg = 0; } - i = SQLITE_CREATE_INDEX; - if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE_CREATE_TEMP_INDEX; - if( sqlite3AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ - goto exit_create_index; + rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0); + if(rc!=SQLITE_OK ){ + goto attach_end; } } -#endif +#endif /* SQLITE_OMIT_AUTHORIZATION */ - /* If pList==0, it means this routine was called to make a primary - ** key out of the last column added to the table under construction. - ** So create a fake list to simulate this. - */ - if( pList==0 ){ - nullId.z = pTab->aCol[pTab->nCol-1].zName; - nullId.n = sqlite3Strlen30((char*)nullId.z); - pList = sqlite3ExprListAppend(pParse, 0, 0); - if( pList==0 ) goto exit_create_index; - sqlite3ExprListSetName(pParse, pList, &nullId, 0); - pList->a[0].sortOrder = (u8)sortOrder; - } - /* Figure out how many bytes of space are required to store explicitly - ** specified collation sequence names. - */ - for(i=0; inExpr; i++){ - Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - CollSeq *pColl = pExpr->pColl; - /* Either pColl!=0 or there was an OOM failure. But if an OOM - ** failure we have quit before reaching this point. */ - if( ALWAYS(pColl) ){ - nExtra += (1 + sqlite3Strlen30(pColl->zName)); - } - } - } + v = sqlite3GetVdbe(pParse); + regArgs = sqlite3GetTempRange(pParse, 4); + sqlite3ExprCode(pParse, pFilename, regArgs); + sqlite3ExprCode(pParse, pDbname, regArgs+1); + sqlite3ExprCode(pParse, pKey, regArgs+2); - /* - ** Allocate the index structure. - */ - nName = sqlite3Strlen30(zName); - nCol = pList->nExpr; - pIndex = sqlite3DbMallocZero(db, - sizeof(Index) + /* Index structure */ - sizeof(int)*nCol + /* Index.aiColumn */ - sizeof(int)*(nCol+1) + /* Index.aiRowEst */ - sizeof(char *)*nCol + /* Index.azColl */ - sizeof(u8)*nCol + /* Index.aSortOrder */ - nName + 1 + /* Index.zName */ - nExtra /* Collation sequence names */ - ); - if( db->mallocFailed ){ - goto exit_create_index; - } - pIndex->azColl = (char**)(&pIndex[1]); - pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]); - pIndex->aiRowEst = (unsigned *)(&pIndex->aiColumn[nCol]); - pIndex->aSortOrder = (u8 *)(&pIndex->aiRowEst[nCol+1]); - pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]); - zExtra = (char *)(&pIndex->zName[nName+1]); - memcpy(pIndex->zName, zName, nName+1); - pIndex->pTable = pTab; - pIndex->nColumn = pList->nExpr; - pIndex->onError = (u8)onError; - pIndex->autoIndex = (u8)(pName==0); - pIndex->pSchema = db->aDb[iDb].pSchema; + assert( v || db->mallocFailed ); + if( v ){ + sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); + assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); + sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); + sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); - /* Check to see if we should honor DESC requests on index columns - */ - if( pDb->pSchema->file_format>=4 ){ - sortOrderMask = -1; /* Honor DESC */ - }else{ - sortOrderMask = 0; /* Ignore DESC */ + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this + ** statement only). For DETACH, set it to false (expire all existing + ** statements). + */ + sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); } + +attach_end: + sqlite3ExprDelete(db, pFilename); + sqlite3ExprDelete(db, pDbname); + sqlite3ExprDelete(db, pKey); +} - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. - ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. - */ - for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName = pListItem->zName; - Column *pTabCol; - int requestedSortOrder; - char *zColl; /* Collation sequence name */ +/* +** Called by the parser to compile a DETACH statement. +** +** DETACH pDbname +*/ +SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ + static const FuncDef detach_func = { + 1, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + detachFunc, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "sqlite_detach", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ + }; + codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); +} - for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; - } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; - } - pIndex->aiColumn[i] = j; - /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of - ** the way the "idxlist" non-terminal is constructed by the parser, - ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl - ** must exist or else there must have been an OOM error. But if there - ** was an OOM error, we would never reach this point. */ - if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ - int nColl; - zColl = pListItem->pExpr->pColl->zName; - nColl = sqlite3Strlen30(zColl) + 1; - assert( nExtra>=nColl ); - memcpy(zExtra, zColl, nColl); - zColl = zExtra; - zExtra += nColl; - nExtra -= nColl; - }else{ - zColl = pTab->aCol[j].zColl; - if( !zColl ){ - zColl = db->pDfltColl->zName; - } - } - if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ - goto exit_create_index; - } - pIndex->azColl[i] = zColl; - requestedSortOrder = pListItem->sortOrder & sortOrderMask; - pIndex->aSortOrder[i] = (u8)requestedSortOrder; - } - sqlite3DefaultRowEst(pIndex); +/* +** Called by the parser to compile an ATTACH statement. +** +** ATTACH p AS pDbname KEY pKey +*/ +SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ + static const FuncDef attach_func = { + 3, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + attachFunc, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "sqlite_attach", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ + }; + codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); +} +#endif /* SQLITE_OMIT_ATTACH */ - if( pTab==pParse->pNewTable ){ - /* This routine has been called to create an automatic index as a - ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or - ** a PRIMARY KEY or UNIQUE clause following the column definitions. - ** i.e. one of: - ** - ** CREATE TABLE t(x PRIMARY KEY, y); - ** CREATE TABLE t(x, y, UNIQUE(x, y)); - ** - ** Either way, check to see if the table already has such an index. If - ** so, don't bother creating this one. This only applies to - ** automatically created indices. Users can do as they wish with - ** explicit indices. - ** - ** Two UNIQUE or PRIMARY KEY constraints are considered equivalent - ** (and thus suppressing the second one) even if they have different - ** sort orders. - ** - ** If there are different collating sequences or if the columns of - ** the constraint occur in different orders, then the constraints are - ** considered distinct and both result in separate indices. - */ - Index *pIdx; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int k; - assert( pIdx->onError!=OE_None ); - assert( pIdx->autoIndex ); - assert( pIndex->onError!=OE_None ); +/* +** Initialize a DbFixer structure. This routine must be called prior +** to passing the structure to one of the sqliteFixAAAA() routines below. +** +** The return value indicates whether or not fixation is required. TRUE +** means we do need to fix the database references, FALSE means we do not. +*/ +SQLITE_PRIVATE int sqlite3FixInit( + DbFixer *pFix, /* The fixer to be initialized */ + Parse *pParse, /* Error messages will be written here */ + int iDb, /* This is the database that must be used */ + const char *zType, /* "view", "trigger", or "index" */ + const Token *pName /* Name of the view, trigger, or index */ +){ + sqlite3 *db; - if( pIdx->nColumn!=pIndex->nColumn ) continue; - for(k=0; knColumn; k++){ - const char *z1; - const char *z2; - if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; - z1 = pIdx->azColl[k]; - z2 = pIndex->azColl[k]; - if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; - } - if( k==pIdx->nColumn ){ - if( pIdx->onError!=pIndex->onError ){ - /* This constraint creates the same index as a previous - ** constraint specified somewhere in the CREATE TABLE statement. - ** However the ON CONFLICT clauses are different. If both this - ** constraint and the previous equivalent constraint have explicit - ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. - */ - if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ - sqlite3ErrorMsg(pParse, - "conflicting ON CONFLICT clauses specified", 0); - } - if( pIdx->onError==OE_Default ){ - pIdx->onError = pIndex->onError; - } - } - goto exit_create_index; - } + if( NEVER(iDb<0) || iDb==1 ) return 0; + db = pParse->db; + assert( db->nDb>iDb ); + pFix->pParse = pParse; + pFix->zDb = db->aDb[iDb].zName; + pFix->zType = zType; + pFix->pName = pName; + return 1; +} + +/* +** The following set of routines walk through the parse tree and assign +** a specific database to all table references where the database name +** was left unspecified in the original SQL statement. The pFix structure +** must have been initialized by a prior call to sqlite3FixInit(). +** +** These routines are used to make sure that an index, trigger, or +** view in one database does not refer to objects in a different database. +** (Exception: indices, triggers, and views in the TEMP database are +** allowed to refer to anything.) If a reference is explicitly made +** to an object in a different database, an error message is added to +** pParse->zErrMsg and these routines return non-zero. If everything +** checks out, these routines return 0. +*/ +SQLITE_PRIVATE int sqlite3FixSrcList( + DbFixer *pFix, /* Context of the fixation */ + SrcList *pList /* The Source list to check and modify */ +){ + int i; + const char *zDb; + struct SrcList_item *pItem; + + if( NEVER(pList==0) ) return 0; + zDb = pFix->zDb; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pItem->zDatabase==0 ){ + pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); + }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->zDatabase); + return 1; } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) + if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; + if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; +#endif } - - /* Link the new Index structure to its table and to the other - ** in-memory database structures. - */ - if( db->init.busy ){ - Index *p; - p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, sqlite3Strlen30(pIndex->zName), - pIndex); - if( p ){ - assert( p==pIndex ); /* Malloc must have failed */ - db->mallocFailed = 1; - goto exit_create_index; + return 0; +} +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE int sqlite3FixSelect( + DbFixer *pFix, /* Context of the fixation */ + Select *pSelect /* The SELECT statement to be fixed to one database */ +){ + while( pSelect ){ + if( sqlite3FixExprList(pFix, pSelect->pEList) ){ + return 1; } - db->flags |= SQLITE_InternChanges; - if( pTblName!=0 ){ - pIndex->tnum = db->init.newTnum; + if( sqlite3FixSrcList(pFix, pSelect->pSrc) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pSelect->pWhere) ){ + return 1; } + if( sqlite3FixExpr(pFix, pSelect->pHaving) ){ + return 1; + } + pSelect = pSelect->pPrior; } - - /* If the db->init.busy is 0 then create the index on disk. This - ** involves writing the index into the master table and filling in the - ** index with the current table contents. - ** - ** The db->init.busy is 0 when the user first enters a CREATE INDEX - ** command. db->init.busy is 1 when a database is opened and - ** CREATE INDEX statements are read out of the master table. In - ** the latter case the index already exists on disk, which is why - ** we don't want to recreate it. - ** - ** If pTblName==0 it means this index is generated as a primary key - ** or UNIQUE constraint of a CREATE TABLE statement. Since the table - ** has just been created, it contains no data and the index initialization - ** step can be skipped. - */ - else{ /* if( db->init.busy==0 ) */ - Vdbe *v; - char *zStmt; - int iMem = ++pParse->nMem; - - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto exit_create_index; - - - /* Create the rootpage for the index - */ - sqlite3BeginWriteOperation(pParse, 1, iDb); - sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); - - /* Gather the complete text of the CREATE INDEX statement into - ** the zStmt variable - */ - if( pStart ){ - assert( pEnd!=0 ); - /* A named index with an explicit CREATE INDEX statement */ - zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", - onError==OE_None ? "" : " UNIQUE", - pEnd->z - pName->z + 1, - pName->z); + return 0; +} +SQLITE_PRIVATE int sqlite3FixExpr( + DbFixer *pFix, /* Context of the fixation */ + Expr *pExpr /* The expression to be fixed to one database */ +){ + while( pExpr ){ + if( ExprHasAnyProperty(pExpr, EP_TokenOnly) ) break; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; }else{ - /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ - /* zStmt = sqlite3MPrintf(""); */ - zStmt = 0; + if( sqlite3FixExprList(pFix, pExpr->x.pList) ) return 1; } - - /* Add an entry in sqlite_master for this index - */ - sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pIndex->zName, - pTab->zName, - iMem, - zStmt - ); - sqlite3DbFree(db, zStmt); - - /* Fill the index with data and reparse the schema. Code an OP_Expire - ** to invalidate all pre-compiled statements. - */ - if( pTblName ){ - sqlite3RefillIndex(pParse, pIndex, iMem); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), - P4_DYNAMIC); - sqlite3VdbeAddOp1(v, OP_Expire, 0); + if( sqlite3FixExpr(pFix, pExpr->pRight) ){ + return 1; } + pExpr = pExpr->pLeft; } - - /* When adding an index to the list of indices for a table, make - ** sure all indices labeled OE_Replace come after all those labeled - ** OE_Ignore. This is necessary for the correct constraint check - ** processing (in sqlite3GenerateConstraintChecks()) as part of - ** UPDATE and INSERT statements. - */ - if( db->init.busy || pTblName==0 ){ - if( onError!=OE_Replace || pTab->pIndex==0 - || pTab->pIndex->onError==OE_Replace){ - pIndex->pNext = pTab->pIndex; - pTab->pIndex = pIndex; - }else{ - Index *pOther = pTab->pIndex; - while( pOther->pNext && pOther->pNext->onError!=OE_Replace ){ - pOther = pOther->pNext; - } - pIndex->pNext = pOther->pNext; - pOther->pNext = pIndex; + return 0; +} +SQLITE_PRIVATE int sqlite3FixExprList( + DbFixer *pFix, /* Context of the fixation */ + ExprList *pList /* The expression to be fixed to one database */ +){ + int i; + struct ExprList_item *pItem; + if( pList==0 ) return 0; + for(i=0, pItem=pList->a; inExpr; i++, pItem++){ + if( sqlite3FixExpr(pFix, pItem->pExpr) ){ + return 1; } - pRet = pIndex; - pIndex = 0; } + return 0; +} +#endif - /* Clean up before exiting */ -exit_create_index: - if( pIndex ){ - sqlite3DbFree(db, pIndex->zColAff); - sqlite3DbFree(db, pIndex); +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE int sqlite3FixTriggerStep( + DbFixer *pFix, /* Context of the fixation */ + TriggerStep *pStep /* The trigger step be fixed to one database */ +){ + while( pStep ){ + if( sqlite3FixSelect(pFix, pStep->pSelect) ){ + return 1; + } + if( sqlite3FixExpr(pFix, pStep->pWhere) ){ + return 1; + } + if( sqlite3FixExprList(pFix, pStep->pExprList) ){ + return 1; + } + pStep = pStep->pNext; } - sqlite3ExprListDelete(db, pList); - sqlite3SrcListDelete(db, pTblName); - sqlite3DbFree(db, zName); - return pRet; + return 0; } +#endif +/************** End of attach.c **********************************************/ +/************** Begin file auth.c ********************************************/ /* -** Fill the Index.aiRowEst[] array with default information - information -** to be used when we have not run the ANALYZE command. +** 2003 January 11 ** -** aiRowEst[0] is suppose to contain the number of elements in the index. -** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the -** number of rows in the table that match any particular value of the -** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns -** of the index. And so forth. It must always be the case that -* -** aiRowEst[N]<=aiRowEst[N-1] -** aiRowEst[N]>=1 +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Apart from that, we have little to go on besides intuition as to -** how aiRowEst[] should be initialized. The numbers generated here -** are based on typical values found in actual indices. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the sqlite3_set_authorizer() +** API. This facility is an optional feature of the library. Embedded +** systems that do not need this facility may omit it by recompiling +** the library with -DSQLITE_OMIT_AUTHORIZATION=1 */ -SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ - unsigned *a = pIdx->aiRowEst; - int i; - unsigned n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nColumn; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nColumn] = 1; - } -} /* -** This routine will drop an existing named index. This routine -** implements the DROP INDEX statement. +** All of the code in this file may be omitted by defining a single +** macro. */ -SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ - Index *pIndex; - Vdbe *v; - sqlite3 *db = pParse->db; - int iDb; - - assert( pParse->nErr==0 ); /* Never called with prior errors */ - if( db->mallocFailed ){ - goto exit_drop_index; - } - assert( pName->nSrc==1 ); - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - goto exit_drop_index; - } - pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); - if( pIndex==0 ){ - if( !ifExists ){ - sqlite3ErrorMsg(pParse, "no such index: %S", pName, 0); - } - pParse->checkSchema = 1; - goto exit_drop_index; - } - if( pIndex->autoIndex ){ - sqlite3ErrorMsg(pParse, "index associated with UNIQUE " - "or PRIMARY KEY constraint cannot be dropped", 0); - goto exit_drop_index; - } - iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); #ifndef SQLITE_OMIT_AUTHORIZATION - { - int code = SQLITE_DROP_INDEX; - Table *pTab = pIndex->pTable; - const char *zDb = db->aDb[iDb].zName; - const char *zTab = SCHEMA_TABLE(iDb); - if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ - goto exit_drop_index; - } - if( !OMIT_TEMPDB && iDb ) code = SQLITE_DROP_TEMP_INDEX; - if( sqlite3AuthCheck(pParse, code, pIndex->zName, pTab->zName, zDb) ){ - goto exit_drop_index; - } - } -#endif - - /* Generate code to remove the index and from the master table */ - v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3BeginWriteOperation(pParse, 1, iDb); - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pIndex->zName - ); - if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.sqlite_stat1 WHERE idx=%Q", - db->aDb[iDb].zName, pIndex->zName - ); - } - sqlite3ChangeCookie(pParse, iDb); - destroyRootPage(pParse, pIndex->tnum, iDb); - sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); - } - -exit_drop_index: - sqlite3SrcListDelete(db, pName); -} /* -** pArray is a pointer to an array of objects. Each object in the -** array is szEntry bytes in size. This routine allocates a new -** object on the end of the array. -** -** *pnEntry is the number of entries already in use. *pnAlloc is -** the previously allocated size of the array. initSize is the -** suggested initial array size allocation. +** Set or clear the access authorization function. ** -** The index of the new entry is returned in *pIdx. +** The access authorization function is be called during the compilation +** phase to verify that the user has read and/or write access permission on +** various fields of the database. The first argument to the auth function +** is a copy of the 3rd argument to this routine. The second argument +** to the auth function is one of these constants: ** -** This routine returns a pointer to the array of objects. This -** might be the same as the pArray parameter or it might be a different -** pointer if the array was resized. -*/ -SQLITE_PRIVATE void *sqlite3ArrayAllocate( - sqlite3 *db, /* Connection to notify of malloc failures */ - void *pArray, /* Array of objects. Might be reallocated */ - int szEntry, /* Size of each object in the array */ - int initSize, /* Suggested initial allocation, in elements */ - int *pnEntry, /* Number of objects currently in use */ - int *pnAlloc, /* Current size of the allocation, in elements */ - int *pIdx /* Write the index of a new slot here */ -){ - char *z; - if( *pnEntry >= *pnAlloc ){ - void *pNew; - int newSize; - newSize = (*pnAlloc)*2 + initSize; - pNew = sqlite3DbRealloc(db, pArray, newSize*szEntry); - if( pNew==0 ){ - *pIdx = -1; - return pArray; - } - *pnAlloc = sqlite3DbMallocSize(db, pNew)/szEntry; - pArray = pNew; - } - z = (char*)pArray; - memset(&z[*pnEntry * szEntry], 0, szEntry); - *pIdx = *pnEntry; - ++*pnEntry; - return pArray; -} - -/* -** Append a new element to the given IdList. Create a new IdList if -** need be. +** SQLITE_CREATE_INDEX +** SQLITE_CREATE_TABLE +** SQLITE_CREATE_TEMP_INDEX +** SQLITE_CREATE_TEMP_TABLE +** SQLITE_CREATE_TEMP_TRIGGER +** SQLITE_CREATE_TEMP_VIEW +** SQLITE_CREATE_TRIGGER +** SQLITE_CREATE_VIEW +** SQLITE_DELETE +** SQLITE_DROP_INDEX +** SQLITE_DROP_TABLE +** SQLITE_DROP_TEMP_INDEX +** SQLITE_DROP_TEMP_TABLE +** SQLITE_DROP_TEMP_TRIGGER +** SQLITE_DROP_TEMP_VIEW +** SQLITE_DROP_TRIGGER +** SQLITE_DROP_VIEW +** SQLITE_INSERT +** SQLITE_PRAGMA +** SQLITE_READ +** SQLITE_SELECT +** SQLITE_TRANSACTION +** SQLITE_UPDATE ** -** A new IdList is returned, or NULL if malloc() fails. +** The third and fourth arguments to the auth function are the name of +** the table and the column that are being accessed. The auth function +** should return either SQLITE_OK, SQLITE_DENY, or SQLITE_IGNORE. If +** SQLITE_OK is returned, it means that access is allowed. SQLITE_DENY +** means that the SQL statement will never-run - the sqlite3_exec() call +** will return with an error. SQLITE_IGNORE means that the SQL statement +** should run but attempts to read the specified column will return NULL +** and attempts to write the column will be ignored. +** +** Setting the auth function to NULL disables this hook. The default +** setting of the auth function is NULL. */ -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ - int i; - if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(IdList) ); - if( pList==0 ) return 0; - pList->nAlloc = 0; - } - pList->a = sqlite3ArrayAllocate( - db, - pList->a, - sizeof(pList->a[0]), - 5, - &pList->nId, - &pList->nAlloc, - &i - ); - if( i<0 ){ - sqlite3IdListDelete(db, pList); - return 0; - } - pList->a[i].zName = sqlite3NameFromToken(db, pToken); - return pList; +SQLITE_API int sqlite3_set_authorizer( + sqlite3 *db, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pArg +){ + sqlite3_mutex_enter(db->mutex); + db->xAuth = xAuth; + db->pAuthArg = pArg; + sqlite3ExpirePreparedStatements(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } /* -** Delete an IdList. +** Write an error message into pParse->zErrMsg that explains that the +** user-supplied authorization function returned an illegal value. */ -SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ - int i; - if( pList==0 ) return; - for(i=0; inId; i++){ - sqlite3DbFree(db, pList->a[i].zName); - } - sqlite3DbFree(db, pList->a); - sqlite3DbFree(db, pList); +static void sqliteAuthBadReturnCode(Parse *pParse){ + sqlite3ErrorMsg(pParse, "authorizer malfunction"); + pParse->rc = SQLITE_ERROR; } /* -** Return the index in pList of the identifier named zId. Return -1 -** if not found. +** Invoke the authorization callback for permission to read column zCol from +** table zTab in database zDb. This function assumes that an authorization +** callback has been registered (i.e. that sqlite3.xAuth is not NULL). +** +** If SQLITE_IGNORE is returned and pExpr is not NULL, then pExpr is changed +** to an SQL NULL expression. Otherwise, if pExpr is NULL, then SQLITE_IGNORE +** is treated as SQLITE_DENY. In this case an error is left in pParse. */ -SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ - int i; - if( pList==0 ) return -1; - for(i=0; inId; i++){ - if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; +SQLITE_PRIVATE int sqlite3AuthReadCol( + Parse *pParse, /* The parser context */ + const char *zTab, /* Table name */ + const char *zCol, /* Column name */ + int iDb /* Index of containing database. */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zName; /* Name of attached database */ + int rc; /* Auth callback return code */ + + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); + if( rc==SQLITE_DENY ){ + if( db->nDb>2 || iDb!=0 ){ + sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); + }else{ + sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol); + } + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ + sqliteAuthBadReturnCode(pParse); } - return -1; + return rc; } /* -** Expand the space allocated for the given SrcList object by -** creating nExtra new slots beginning at iStart. iStart is zero based. -** New slots are zeroed. -** -** For example, suppose a SrcList initially contains two entries: A,B. -** To append 3 new entries onto the end, do this: -** -** sqlite3SrcListEnlarge(db, pSrclist, 3, 2); -** -** After the call above it would contain: A, B, nil, nil, nil. -** If the iStart argument had been 1 instead of 2, then the result -** would have been: A, nil, nil, nil, B. To prepend the new slots, -** the iStart value would be 0. The result then would -** be: nil, nil, nil, A, B. +** The pExpr should be a TK_COLUMN expression. The table referred to +** is in pTabList or else it is the NEW or OLD table of a trigger. +** Check to see if it is OK to read this particular column. ** -** If a memory allocation fails the SrcList is unchanged. The -** db->mallocFailed flag will be set to true. +** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN +** instruction into a TK_NULL. If the auth function returns SQLITE_DENY, +** then generate an error. */ -SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( - sqlite3 *db, /* Database connection to notify of OOM errors */ - SrcList *pSrc, /* The SrcList to be enlarged */ - int nExtra, /* Number of new slots to add to pSrc->a[] */ - int iStart /* Index in pSrc->a[] of first new slot */ +SQLITE_PRIVATE void sqlite3AuthRead( + Parse *pParse, /* The parser context */ + Expr *pExpr, /* The expression to check authorization on */ + Schema *pSchema, /* The schema of the expression */ + SrcList *pTabList /* All table that pExpr might refer to */ ){ - int i; + sqlite3 *db = pParse->db; + Table *pTab = 0; /* The table being read */ + const char *zCol; /* Name of the column of the table */ + int iSrc; /* Index in pTabList->a[] of table being read */ + int iDb; /* The index of the database the expression refers to */ + int iCol; /* Index of column in table */ - /* Sanity checking on calling parameters */ - assert( iStart>=0 ); - assert( nExtra>=1 ); - assert( pSrc!=0 ); - assert( iStart<=pSrc->nSrc ); + if( db->xAuth==0 ) return; + iDb = sqlite3SchemaToIndex(pParse->db, pSchema); + if( iDb<0 ){ + /* An attempt to read a column out of a subquery or other + ** temporary table. */ + return; + } - /* Allocate additional space if needed */ - if( pSrc->nSrc+nExtra>pSrc->nAlloc ){ - SrcList *pNew; - int nAlloc = pSrc->nSrc+nExtra; - int nGot; - pNew = sqlite3DbRealloc(db, pSrc, - sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); - if( pNew==0 ){ - assert( db->mallocFailed ); - return pSrc; + assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); + if( pExpr->op==TK_TRIGGER ){ + pTab = pParse->pTriggerTab; + }else{ + assert( pTabList ); + for(iSrc=0; ALWAYS(iSrcnSrc); iSrc++){ + if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ + pTab = pTabList->a[iSrc].pTab; + break; + } } - pSrc = pNew; - nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = (u16)nGot; } + iCol = pExpr->iColumn; + if( NEVER(pTab==0) ) return; - /* Move existing slots that come after the newly inserted slots - ** out of the way */ - for(i=pSrc->nSrc-1; i>=iStart; i--){ - pSrc->a[i+nExtra] = pSrc->a[i]; + if( iCol>=0 ){ + assert( iColnCol ); + zCol = pTab->aCol[iCol].zName; + }else if( pTab->iPKey>=0 ){ + assert( pTab->iPKeynCol ); + zCol = pTab->aCol[pTab->iPKey].zName; + }else{ + zCol = "ROWID"; } - pSrc->nSrc += (i16)nExtra; - - /* Zero the newly allocated slots */ - memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); - for(i=iStart; ia[i].iCursor = -1; + assert( iDb>=0 && iDbnDb ); + if( SQLITE_IGNORE==sqlite3AuthReadCol(pParse, pTab->zName, zCol, iDb) ){ + pExpr->op = TK_NULL; } - - /* Return a pointer to the enlarged SrcList */ - return pSrc; } - /* -** Append a new table name to the given SrcList. Create a new SrcList if -** need be. A new entry is created in the SrcList even if pTable is NULL. -** -** A SrcList is returned, or NULL if there is an OOM error. The returned -** SrcList might be the same as the SrcList that was input or it might be -** a new one. If an OOM error does occurs, then the prior value of pList -** that is input to this routine is automatically freed. -** -** If pDatabase is not null, it means that the table has an optional -** database name prefix. Like this: "database.table". The pDatabase -** points to the table name and the pTable points to the database name. -** The SrcList.a[].zName field is filled with the table name which might -** come from pTable (if pDatabase is NULL) or from pDatabase. -** SrcList.a[].zDatabase is filled with the database name from pTable, -** or with NULL if no database is specified. -** -** In other words, if call like this: -** -** sqlite3SrcListAppend(D,A,B,0); -** -** Then B is a table name and the database name is unspecified. If called -** like this: -** -** sqlite3SrcListAppend(D,A,B,C); -** -** Then C is the table name and B is the database name. If C is defined -** then so is B. In other words, we never have a case where: -** -** sqlite3SrcListAppend(D,A,0,C); -** -** Both pTable and pDatabase are assumed to be quoted. They are dequoted -** before being added to the SrcList. +** Do an authorization check using the code and arguments given. Return +** either SQLITE_OK (zero) or SQLITE_IGNORE or SQLITE_DENY. If SQLITE_DENY +** is returned, then the error count and error message in pParse are +** modified appropriately. */ -SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( - sqlite3 *db, /* Connection to notify of malloc failures */ - SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ - Token *pTable, /* Table to append */ - Token *pDatabase /* Database of the table */ +SQLITE_PRIVATE int sqlite3AuthCheck( + Parse *pParse, + int code, + const char *zArg1, + const char *zArg2, + const char *zArg3 ){ - struct SrcList_item *pItem; - assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ - if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); - if( pList==0 ) return 0; - pList->nAlloc = 1; - } - pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); - if( db->mallocFailed ){ - sqlite3SrcListDelete(db, pList); - return 0; + sqlite3 *db = pParse->db; + int rc; + + /* Don't do any authorization checks if the database is initialising + ** or if the parser is being invoked from within sqlite3_declare_vtab. + */ + if( db->init.busy || IN_DECLARE_VTAB ){ + return SQLITE_OK; } - pItem = &pList->a[pList->nSrc-1]; - if( pDatabase && pDatabase->z==0 ){ - pDatabase = 0; + + if( db->xAuth==0 ){ + return SQLITE_OK; } - if( pDatabase ){ - Token *pTemp = pDatabase; - pDatabase = pTable; - pTable = pTemp; + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); + if( rc==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized"); + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){ + rc = SQLITE_DENY; + sqliteAuthBadReturnCode(pParse); } - pItem->zName = sqlite3NameFromToken(db, pTable); - pItem->zDatabase = sqlite3NameFromToken(db, pDatabase); - return pList; + return rc; } /* -** Assign VdbeCursor index numbers to all tables in a SrcList +** Push an authorization context. After this routine is called, the +** zArg3 argument to authorization callbacks will be zContext until +** popped. Or if pParse==0, this routine is a no-op. */ -SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ - int i; - struct SrcList_item *pItem; - assert(pList || pParse->db->mallocFailed ); - if( pList ){ - for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->iCursor>=0 ) break; - pItem->iCursor = pParse->nTab++; - if( pItem->pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); - } - } - } +SQLITE_PRIVATE void sqlite3AuthContextPush( + Parse *pParse, + AuthContext *pContext, + const char *zContext +){ + assert( pParse ); + pContext->pParse = pParse; + pContext->zAuthContext = pParse->zAuthContext; + pParse->zAuthContext = zContext; } /* -** Delete an entire SrcList including all its substructure. +** Pop an authorization context that was previously pushed +** by sqlite3AuthContextPush */ -SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ - int i; - struct SrcList_item *pItem; - if( pList==0 ) return; - for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - sqlite3DbFree(db, pItem->zDatabase); - sqlite3DbFree(db, pItem->zName); - sqlite3DbFree(db, pItem->zAlias); - sqlite3DbFree(db, pItem->zIndex); - sqlite3DeleteTable(db, pItem->pTab); - sqlite3SelectDelete(db, pItem->pSelect); - sqlite3ExprDelete(db, pItem->pOn); - sqlite3IdListDelete(db, pItem->pUsing); +SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ + if( pContext->pParse ){ + pContext->pParse->zAuthContext = pContext->zAuthContext; + pContext->pParse = 0; } - sqlite3DbFree(db, pList); } +#endif /* SQLITE_OMIT_AUTHORIZATION */ + +/************** End of auth.c ************************************************/ +/************** Begin file build.c *******************************************/ /* -** This routine is called by the parser to add a new term to the -** end of a growing FROM clause. The "p" parameter is the part of -** the FROM clause that has already been constructed. "p" is NULL -** if this is the first term of the FROM clause. pTable and pDatabase -** are the name of the table and database named in the FROM clause term. -** pDatabase is NULL if the database name qualifier is missing - the -** usual case. If the term has a alias, then pAlias points to the -** alias token. If the term is a subquery, then pSubquery is the -** SELECT statement that the subquery encodes. The pTable and -** pDatabase parameters are NULL for subqueries. The pOn and pUsing -** parameters are the content of the ON and USING clauses. +** 2001 September 15 ** -** Return a new SrcList which encodes is the FROM with the new -** term added. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the SQLite parser +** when syntax rules are reduced. The routines in this file handle the +** following kinds of SQL syntax: +** +** CREATE TABLE +** DROP TABLE +** CREATE INDEX +** DROP INDEX +** creating ID lists +** BEGIN TRANSACTION +** COMMIT +** ROLLBACK */ -SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( - Parse *pParse, /* Parsing context */ - SrcList *p, /* The left part of the FROM clause already seen */ - Token *pTable, /* Name of the table to add to the FROM clause */ - Token *pDatabase, /* Name of the database containing pTable */ - Token *pAlias, /* The right-hand side of the AS subexpression */ - Select *pSubquery, /* A subquery used in place of a table name */ - Expr *pOn, /* The ON clause of a join */ - IdList *pUsing /* The USING clause of a join */ -){ - struct SrcList_item *pItem; - sqlite3 *db = pParse->db; - if( !p && (pOn || pUsing) ){ - sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", - (pOn ? "ON" : "USING") - ); - goto append_from_error; - } - p = sqlite3SrcListAppend(db, p, pTable, pDatabase); - if( p==0 || NEVER(p->nSrc==0) ){ - goto append_from_error; - } - pItem = &p->a[p->nSrc-1]; - assert( pAlias!=0 ); - if( pAlias->n ){ - pItem->zAlias = sqlite3NameFromToken(db, pAlias); - } - pItem->pSelect = pSubquery; - pItem->pOn = pOn; - pItem->pUsing = pUsing; - return p; - append_from_error: - assert( p==0 ); - sqlite3ExprDelete(db, pOn); - sqlite3IdListDelete(db, pUsing); - sqlite3SelectDelete(db, pSubquery); - return 0; +/* +** This routine is called when a new SQL statement is beginning to +** be parsed. Initialize the pParse structure as needed. +*/ +SQLITE_PRIVATE void sqlite3BeginParse(Parse *pParse, int explainFlag){ + pParse->explain = (u8)explainFlag; + pParse->nVar = 0; } +#ifndef SQLITE_OMIT_SHARED_CACHE /* -** Add an INDEXED BY or NOT INDEXED clause to the most recently added -** element of the source-list passed as the second argument. +** The TableLock structure is only used by the sqlite3TableLock() and +** codeTableLocks() functions. */ -SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ - assert( pIndexedBy!=0 ); - if( p && ALWAYS(p->nSrc>0) ){ - struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->notIndexed==0 && pItem->zIndex==0 ); - if( pIndexedBy->n==1 && !pIndexedBy->z ){ - /* A "NOT INDEXED" clause was supplied. See parse.y - ** construct "indexed_opt" for details. */ - pItem->notIndexed = 1; - }else{ - pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); - } - } -} +struct TableLock { + int iDb; /* The database containing the table to be locked */ + int iTab; /* The root page of the table to be locked */ + u8 isWriteLock; /* True for write lock. False for a read lock */ + const char *zName; /* Name of the table */ +}; /* -** When building up a FROM clause in the parser, the join operator -** is initially attached to the left operand. But the code generator -** expects the join operator to be on the right operand. This routine -** Shifts all join operators from left to right for an entire FROM -** clause. -** -** Example: Suppose the join is like this: +** Record the fact that we want to lock a table at run-time. ** -** A natural cross join B +** The table to be locked has root page iTab and is found in database iDb. +** A read or a write lock can be taken depending on isWritelock. ** -** The operator is "natural cross join". The A and B operands are stored -** in p->a[0] and p->a[1], respectively. The parser initially stores the -** operator with A. This routine shifts that operator over to B. +** This routine just records the fact that the lock is desired. The +** code to make the lock occur is generated by a later call to +** codeTableLocks() which occurs during sqlite3FinishCoding(). */ -SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ - if( p && p->a ){ - int i; - for(i=p->nSrc-1; i>0; i--){ - p->a[i].jointype = p->a[i-1].jointype; - } - p->a[0].jointype = 0; - } -} - -/* -** Begin a transaction -*/ -SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ - sqlite3 *db; - Vdbe *v; +SQLITE_PRIVATE void sqlite3TableLock( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database containing the table to lock */ + int iTab, /* Root page number of the table to be locked */ + u8 isWriteLock, /* True for a write lock */ + const char *zName /* Name of the table to be locked */ +){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); int i; + int nBytes; + TableLock *p; + assert( iDb>=0 ); - assert( pParse!=0 ); - db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ - if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ - return; - } - v = sqlite3GetVdbe(pParse); - if( !v ) return; - if( type!=TK_DEFERRED ){ - for(i=0; inDb; i++){ - sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); - sqlite3VdbeUsesBtree(v, i); + for(i=0; inTableLock; i++){ + p = &pToplevel->aTableLock[i]; + if( p->iDb==iDb && p->iTab==iTab ){ + p->isWriteLock = (p->isWriteLock || isWriteLock); + return; } } - sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); + + nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); + pToplevel->aTableLock = + sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); + if( pToplevel->aTableLock ){ + p = &pToplevel->aTableLock[pToplevel->nTableLock++]; + p->iDb = iDb; + p->iTab = iTab; + p->isWriteLock = isWriteLock; + p->zName = zName; + }else{ + pToplevel->nTableLock = 0; + pToplevel->db->mallocFailed = 1; + } } /* -** Commit a transaction +** Code an OP_TableLock instruction for each table locked by the +** statement (configured by calls to sqlite3TableLock()). */ -SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ - sqlite3 *db; - Vdbe *v; +static void codeTableLocks(Parse *pParse){ + int i; + Vdbe *pVdbe; - assert( pParse!=0 ); - db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ - if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ){ - return; - } - v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); + pVdbe = sqlite3GetVdbe(pParse); + assert( pVdbe!=0 ); /* sqlite3GetVdbe cannot fail: VDBE already allocated */ + + for(i=0; inTableLock; i++){ + TableLock *p = &pParse->aTableLock[i]; + int p1 = p->iDb; + sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, + p->zName, P4_STATIC); } } +#else + #define codeTableLocks(x) +#endif /* -** Rollback a transaction +** This routine is called after a single SQL statement has been +** parsed and a VDBE program to execute that statement has been +** prepared. This routine puts the finishing touches on the +** VDBE program and resets the pParse structure for the next +** parse. +** +** Note that if an error occurred, it might be the case that +** no VDBE code was generated. */ -SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ +SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; - assert( pParse!=0 ); db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ - if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ){ - return; - } + if( db->mallocFailed ) return; + if( pParse->nested ) return; + if( pParse->nErr ) return; + + /* Begin by generating some termination code at the end of the + ** vdbe program + */ v = sqlite3GetVdbe(pParse); + assert( !pParse->isMultiWrite + || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ - sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 1); - } -} + sqlite3VdbeAddOp0(v, OP_Halt); -/* -** This function is called by the parser when it parses a command to create, -** release or rollback an SQL savepoint. -*/ -SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ - char *zName = sqlite3NameFromToken(pParse->db, pName); - if( zName ){ - Vdbe *v = sqlite3GetVdbe(pParse); -#ifndef SQLITE_OMIT_AUTHORIZATION - static const char * const az[] = { "BEGIN", "RELEASE", "ROLLBACK" }; - assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 ); + /* The cookie mask contains one bit for each database file open. + ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are + ** set for each database that is used. Generate code to start a + ** transaction on each used database and to verify the schema cookie + ** on each used database. + */ + if( pParse->cookieGoto>0 ){ + yDbMask mask; + int iDb; + sqlite3VdbeJumpHere(v, pParse->cookieGoto-1); + for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ + if( (mask & pParse->cookieMask)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v,OP_Transaction, iDb, (mask & pParse->writeMask)!=0); + if( db->init.busy==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + sqlite3VdbeAddOp3(v, OP_VerifyCookie, + iDb, pParse->cookieValue[iDb], + db->aDb[iDb].pSchema->iGeneration); + } + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + { + int i; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; + } #endif - if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){ - sqlite3DbFree(pParse->db, zName); - return; + + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); + + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pParse->cookieGoto); } - sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC); } + + + /* Get the VDBE program ready for execution + */ + if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ +#ifdef SQLITE_DEBUG + FILE *trace = (db->flags & SQLITE_VdbeTrace)!=0 ? stdout : 0; + sqlite3VdbeTrace(v, trace); +#endif + assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ + /* A minimum of one cursor is required if autoincrement is used + * See ticket [a696379c1f08866] */ + if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); + pParse->rc = SQLITE_DONE; + pParse->colNamesSet = 0; + }else{ + pParse->rc = SQLITE_ERROR; + } + pParse->nTab = 0; + pParse->nMem = 0; + pParse->nSet = 0; + pParse->nVar = 0; + pParse->cookieMask = 0; + pParse->cookieGoto = 0; } /* -** Make sure the TEMP database is open and available for use. Return -** the number of errors. Leave any error messages in the pParse structure. +** Run the parser and code generator recursively in order to generate +** code for the SQL statement given onto the end of the pParse context +** currently under construction. When the parser is run recursively +** this way, the final OP_Halt is not appended and other initialization +** and finalization steps are omitted because those are handling by the +** outermost parser. +** +** Not everything is nestable. This facility is designed to permit +** INSERT, UPDATE, and DELETE operations against SQLITE_MASTER. Use +** care if you decide to try to use this routine for some other purposes. */ -SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ +SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ + va_list ap; + char *zSql; + char *zErrMsg = 0; sqlite3 *db = pParse->db; - if( db->aDb[1].pBt==0 && !pParse->explain ){ - int rc; - Btree *pBt; - static const int flags = - SQLITE_OPEN_READWRITE | - SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_TEMP_DB; +# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) + char saveBuf[SAVE_SZ]; - rc = sqlite3BtreeOpen(0, db, &pBt, 0, flags); - if( rc!=SQLITE_OK ){ - sqlite3ErrorMsg(pParse, "unable to open a temporary database " - "file for storing temporary tables"); - pParse->rc = rc; - return 1; - } - db->aDb[1].pBt = pBt; - assert( db->aDb[1].pSchema ); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ - db->mallocFailed = 1; - return 1; - } + if( pParse->nErr ) return; + assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ + va_start(ap, zFormat); + zSql = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + if( zSql==0 ){ + return; /* A malloc must have failed */ } - return 0; + pParse->nested++; + memcpy(saveBuf, &pParse->nVar, SAVE_SZ); + memset(&pParse->nVar, 0, SAVE_SZ); + sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3DbFree(db, zErrMsg); + sqlite3DbFree(db, zSql); + memcpy(&pParse->nVar, saveBuf, SAVE_SZ); + pParse->nested--; } /* -** Generate VDBE code that will verify the schema cookie and start -** a read-transaction for all named database files. -** -** It is important that all schema cookies be verified and all -** read transactions be started before anything else happens in -** the VDBE program. But this routine can be called after much other -** code has been generated. So here is what we do: +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. ** -** The first time this routine is called, we code an OP_Goto that -** will jump to a subroutine at the end of the program. Then we -** record every database that needs its schema verified in the -** pParse->cookieMask field. Later, after all other code has been -** generated, the subroutine that does the cookie verifications and -** starts the transactions will be coded and the OP_Goto P2 value -** will be made to point to that subroutine. The generation of the -** cookie verification subroutine code happens in sqlite3FinishCoding(). +** If zDatabase is 0, all databases are searched for the table and the +** first matching table is returned. (No checking for duplicate table +** names is done.) The search order is TEMP first, then MAIN, then any +** auxiliary databases added using the ATTACH command. ** -** If iDb<0 then code the OP_Goto only - don't set flag to verify the -** schema on any databases. This can be used to position the OP_Goto -** early in the code, before we know if any database tables will be used. +** See also sqlite3LocateTable(). */ -SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - - if( pToplevel->cookieGoto==0 ){ - Vdbe *v = sqlite3GetVdbe(pToplevel); - if( v==0 ) return; /* This only happens if there was a prior error */ - pToplevel->cookieGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0)+1; - } - if( iDb>=0 ){ - sqlite3 *db = pToplevel->db; - int mask; - - assert( iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); - assert( iDbcookieMask & mask)==0 ){ - pToplevel->cookieMask |= mask; - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; - if( !OMIT_TEMPDB && iDb==1 ){ - sqlite3OpenTempDatabase(pToplevel); - } - } +SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ + Table *p = 0; + int i; + int nName; + assert( zName!=0 ); + nName = sqlite3Strlen30(zName); + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); + if( p ) break; } + return p; } /* -** Generate VDBE code that prepares for doing an operation that -** might change the database. +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. Also leave an +** error message in pParse->zErrMsg. ** -** This routine starts a new transaction if we are not already within -** a transaction. If we are already within a transaction, then a checkpoint -** is set if the setStatement parameter is true. A checkpoint should -** be set for operations that might fail (due to a constraint) part of -** the way through and which will need to undo some writes without having to -** rollback the whole transaction. For operations where all constraints -** can be checked before any changes are made to the database, it is never -** necessary to undo a write and the checkpoint should not be set. +** The difference between this routine and sqlite3FindTable() is that this +** routine leaves an error message in pParse->zErrMsg where +** sqlite3FindTable() does not. */ -SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3CodeVerifySchema(pParse, iDb); - pToplevel->writeMask |= 1<isMultiWrite |= setStatement; -} +SQLITE_PRIVATE Table *sqlite3LocateTable( + Parse *pParse, /* context in which to report errors */ + int isView, /* True if looking for a VIEW rather than a TABLE */ + const char *zName, /* Name of the table we are looking for */ + const char *zDbase /* Name of the database. Might be NULL */ +){ + Table *p; -/* -** Indicate that the statement currently under construction might write -** more than one entry (example: deleting one row then inserting another, -** inserting multiple rows in a table, or inserting a row and index entries.) -** If an abort occurs after some of these writes have completed, then it will -** be necessary to undo the completed writes. -*/ -SQLITE_PRIVATE void sqlite3MultiWrite(Parse *pParse){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - pToplevel->isMultiWrite = 1; -} + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return 0; + } -/* -** The code generator calls this routine if is discovers that it is -** possible to abort a statement prior to completion. In order to -** perform this abort without corrupting the database, we need to make -** sure that the statement is protected by a statement transaction. -** -** Technically, we only need to set the mayAbort flag if the -** isMultiWrite flag was previously set. There is a time dependency -** such that the abort must occur after the multiwrite. This makes -** some statements involving the REPLACE conflict resolution algorithm -** go a little faster. But taking advantage of this time dependency -** makes it more difficult to prove that the code is correct (in -** particular, it prevents us from writing an effective -** implementation of sqlite3AssertMayAbort()) and so we have chosen -** to take the safe route and skip the optimization. -*/ -SQLITE_PRIVATE void sqlite3MayAbort(Parse *pParse){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - pToplevel->mayAbort = 1; + p = sqlite3FindTable(pParse->db, zName, zDbase); + if( p==0 ){ + const char *zMsg = isView ? "no such view" : "no such table"; + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + pParse->checkSchema = 1; + } + return p; } /* -** Code an OP_Halt that causes the vdbe to return an SQLITE_CONSTRAINT -** error. The onError parameter determines which (if any) of the statement -** and/or current transaction is rolled back. +** Locate the in-memory structure that describes +** a particular index given the name of that index +** and the name of the database that contains the index. +** Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the +** table and the first matching index is returned. (No checking +** for duplicate index names is done.) The search order is +** TEMP first, then MAIN, then any auxiliary databases added +** using the ATTACH command. */ -SQLITE_PRIVATE void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ - Vdbe *v = sqlite3GetVdbe(pParse); - if( onError==OE_Abort ){ - sqlite3MayAbort(pParse); +SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ + Index *p = 0; + int i; + int nName = sqlite3Strlen30(zName); + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + Schema *pSchema = db->aDb[j].pSchema; + assert( pSchema ); + if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&pSchema->idxHash, zName, nName); + if( p ) break; } - sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); + return p; } /* -** Check to see if pIndex uses the collating sequence pColl. Return -** true if it does and false if it does not. +** Reclaim the memory used by an index */ -#ifndef SQLITE_OMIT_REINDEX -static int collationMatch(const char *zColl, Index *pIndex){ - int i; - assert( zColl!=0 ); - for(i=0; inColumn; i++){ - const char *z = pIndex->azColl[i]; - assert( z!=0 ); - if( 0==sqlite3StrICmp(z, zColl) ){ - return 1; - } - } - return 0; -} +static void freeIndex(sqlite3 *db, Index *p){ +#ifndef SQLITE_OMIT_ANALYZE + sqlite3DeleteIndexSamples(db, p); #endif + sqlite3DbFree(db, p->zColAff); + sqlite3DbFree(db, p); +} /* -** Recompute all indices of pTab that use the collating sequence pColl. -** If pColl==0 then recompute all indices of pTab. +** For the index called zIdxName which is found in the database iDb, +** unlike that index from its Table then remove the index from +** the index hash table and free all memory structures associated +** with the index. */ -#ifndef SQLITE_OMIT_REINDEX -static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ - Index *pIndex; /* An index associated with pTab */ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ + Index *pIndex; + int len; + Hash *pHash; - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( zColl==0 || collationMatch(zColl, pIndex) ){ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &db->aDb[iDb].pSchema->idxHash; + len = sqlite3Strlen30(zIdxName); + pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); + if( ALWAYS(pIndex) ){ + if( pIndex->pTable->pIndex==pIndex ){ + pIndex->pTable->pIndex = pIndex->pNext; + }else{ + Index *p; + /* Justification of ALWAYS(); The index must be on the list of + ** indices. */ + p = pIndex->pTable->pIndex; + while( ALWAYS(p) && p->pNext!=pIndex ){ p = p->pNext; } + if( ALWAYS(p && p->pNext==pIndex) ){ + p->pNext = pIndex->pNext; + } } + freeIndex(db, pIndex); } + db->flags |= SQLITE_InternChanges; } -#endif /* -** Recompute all indices of all tables in all databases where the -** indices use the collating sequence pColl. If pColl==0 then recompute -** all indices everywhere. +** Erase all schema information from the in-memory hash tables of +** a single database. This routine is called to reclaim memory +** before the database closes. It is also called during a rollback +** if there were schema changes during the transaction or if a +** schema-cookie mismatch occurs. +** +** If iDb<0 then reset the internal schema tables for all database +** files. If iDb>=0 then reset the internal schema for only the +** single file indicated. */ -#ifndef SQLITE_OMIT_REINDEX -static void reindexDatabases(Parse *pParse, char const *zColl){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - sqlite3 *db = pParse->db; /* The database connection */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ +SQLITE_PRIVATE void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ + int i, j; + assert( iDbnDb ); - for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ - assert( pDb!=0 ); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - reindexTable(pParse, pTab, zColl); + if( iDb>=0 ){ + /* Case 1: Reset the single schema identified by iDb */ + Db *pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pDb->pSchema!=0 ); + sqlite3SchemaClear(pDb->pSchema); + + /* If any database other than TEMP is reset, then also reset TEMP + ** since TEMP might be holding triggers that reference tables in the + ** other database. + */ + if( iDb!=1 ){ + pDb = &db->aDb[1]; + assert( pDb->pSchema!=0 ); + sqlite3SchemaClear(pDb->pSchema); } + return; } -} -#endif + /* Case 2 (from here to the end): Reset all schemas for all attached + ** databases. */ + assert( iDb<0 ); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pSchema ){ + sqlite3SchemaClear(pDb->pSchema); + } + } + db->flags &= ~SQLITE_InternChanges; + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); -/* -** Generate code for the REINDEX command. -** -** REINDEX -- 1 -** REINDEX -- 2 -** REINDEX ?.? -- 3 -** REINDEX ?.? -- 4 -** -** Form 1 causes all indices in all attached databases to be rebuilt. -** Form 2 rebuilds all indices in all databases that use the named -** collating function. Forms 3 and 4 rebuild the named index or all -** indices associated with the named table. -*/ -#ifndef SQLITE_OMIT_REINDEX -SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ - CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ - char *z; /* Name of a table or index */ - const char *zDb; /* Name of the database */ - Table *pTab; /* A table in the database */ - Index *pIndex; /* An index associated with pTab */ - int iDb; /* The database index number */ - sqlite3 *db = pParse->db; /* The database connection */ - Token *pObjName; /* Name of the table or index to be reindexed */ - - /* Read the database schema. If an error occurs, leave an error message - ** and code in pParse and return NULL. */ - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - return; - } - - if( pName1==0 ){ - reindexDatabases(pParse, 0); - return; - }else if( NEVER(pName2==0) || pName2->z==0 ){ - char *zColl; - assert( pName1->z ); - zColl = sqlite3NameFromToken(pParse->db, pName1); - if( !zColl ) return; - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - if( pColl ){ - reindexDatabases(pParse, zColl); - sqlite3DbFree(db, zColl); - return; + /* If one or more of the auxiliary database files has been closed, + ** then remove them from the auxiliary database list. We take the + ** opportunity to do this here since we have just deleted all of the + ** schema hash tables and therefore do not have to make any changes + ** to any of those tables. + */ + for(i=j=2; inDb; i++){ + struct Db *pDb = &db->aDb[i]; + if( pDb->pBt==0 ){ + sqlite3DbFree(db, pDb->zName); + pDb->zName = 0; + continue; } - sqlite3DbFree(db, zColl); - } - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); - if( iDb<0 ) return; - z = sqlite3NameFromToken(db, pObjName); - if( z==0 ) return; - zDb = db->aDb[iDb].zName; - pTab = sqlite3FindTable(db, z, zDb); - if( pTab ){ - reindexTable(pParse, pTab, 0); - sqlite3DbFree(db, z); - return; + if( jaDb[j] = db->aDb[i]; + } + j++; } - pIndex = sqlite3FindIndex(db, z, zDb); - sqlite3DbFree(db, z); - if( pIndex ){ - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - return; + memset(&db->aDb[j], 0, (db->nDb-j)*sizeof(db->aDb[j])); + db->nDb = j; + if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ + memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); + sqlite3DbFree(db, db->aDb); + db->aDb = db->aDbStatic; } - sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); } -#endif /* -** Return a dynamicly allocated KeyInfo structure that can be used -** with OP_OpenRead or OP_OpenWrite to access database index pIdx. -** -** If successful, a pointer to the new structure is returned. In this case -** the caller is responsible for calling sqlite3DbFree(db, ) on the returned -** pointer. If an error occurs (out of memory or missing collation -** sequence), NULL is returned and the state of pParse updated to reflect -** the error. +** This routine is called when a commit occurs. */ -SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ - int i; - int nCol = pIdx->nColumn; - int nBytes = sizeof(KeyInfo) + (nCol-1)*sizeof(CollSeq*) + nCol; - sqlite3 *db = pParse->db; - KeyInfo *pKey = (KeyInfo *)sqlite3DbMallocZero(db, nBytes); +SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ + db->flags &= ~SQLITE_InternChanges; +} - if( pKey ){ - pKey->db = pParse->db; - pKey->aSortOrder = (u8 *)&(pKey->aColl[nCol]); - assert( &pKey->aSortOrder[nCol]==&(((u8 *)pKey)[nBytes]) ); - for(i=0; iazColl[i]; - assert( zColl ); - pKey->aColl[i] = sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; +/* +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). +*/ +static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ + int i; + Column *pCol; + assert( pTable!=0 ); + if( (pCol = pTable->aCol)!=0 ){ + for(i=0; inCol; i++, pCol++){ + sqlite3DbFree(db, pCol->zName); + sqlite3ExprDelete(db, pCol->pDflt); + sqlite3DbFree(db, pCol->zDflt); + sqlite3DbFree(db, pCol->zType); + sqlite3DbFree(db, pCol->zColl); } - pKey->nField = (u16)nCol; - } - - if( pParse->nErr ){ - sqlite3DbFree(db, pKey); - pKey = 0; + sqlite3DbFree(db, pTable->aCol); } - return pKey; } -/************** End of build.c ***********************************************/ -/************** Begin file callback.c ****************************************/ /* -** 2005 May 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* +** Remove the memory data structures associated with the given +** Table. No changes are made to disk by this routine. ** -** This file contains functions used to access the internal hash tables -** of user defined functions and collation sequences. +** This routine just deletes the data structure. It does not unlink +** the table data structure from the hash table. But it does destroy +** memory structures of the indices and foreign keys associated with +** the table. */ +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + Index *pIndex, *pNext; + assert( !pTable || pTable->nRef>0 ); -/* -** Invoke the 'collation needed' callback to request a collation sequence -** in the encoding enc of name zName, length nName. -*/ -static void callCollNeeded(sqlite3 *db, int enc, const char *zName){ - assert( !db->xCollNeeded || !db->xCollNeeded16 ); - if( db->xCollNeeded ){ - char *zExternal = sqlite3DbStrDup(db, zName); - if( !zExternal ) return; - db->xCollNeeded(db->pCollNeededArg, db, enc, zExternal); - sqlite3DbFree(db, zExternal); - } -#ifndef SQLITE_OMIT_UTF16 - if( db->xCollNeeded16 ){ - char const *zExternal; - sqlite3_value *pTmp = sqlite3ValueNew(db); - sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF8, SQLITE_STATIC); - zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE); - if( zExternal ){ - db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + + /* Delete all indices associated with this table. */ + for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ + pNext = pIndex->pNext; + assert( pIndex->pSchema==pTable->pSchema ); + if( !db || db->pnBytesFreed==0 ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + ); + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + assert( pOld==pIndex || pOld==0 ); } - sqlite3ValueFree(pTmp); + freeIndex(db, pIndex); } + + /* Delete any foreign keys attached to this table. */ + sqlite3FkDelete(db, pTable); + + /* Delete the Table structure itself. + */ + sqliteDeleteColumnNames(db, pTable); + sqlite3DbFree(db, pTable->zName); + sqlite3DbFree(db, pTable->zColAff); + sqlite3SelectDelete(db, pTable->pSelect); +#ifndef SQLITE_OMIT_CHECK + sqlite3ExprDelete(db, pTable->pCheck); +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3VtabClear(db, pTable); #endif + sqlite3DbFree(db, pTable); } /* -** This routine is called if the collation factory fails to deliver a -** collation function in the best encoding but there may be other versions -** of this collation function (for other text encodings) available. Use one -** of these instead if they exist. Avoid a UTF-8 <-> UTF-16 conversion if -** possible. +** Unlink the given table from the hash tables and the delete the +** table structure with all its indices and foreign keys. */ -static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ - CollSeq *pColl2; - char *z = pColl->zName; - int i; - static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 }; - for(i=0; i<3; i++){ - pColl2 = sqlite3FindCollSeq(db, aEnc[i], z, 0); - if( pColl2->xCmp!=0 ){ - memcpy(pColl, pColl2, sizeof(CollSeq)); - pColl->xDel = 0; /* Do not copy the destructor */ - return SQLITE_OK; - } - } - return SQLITE_ERROR; +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ + Table *p; + Db *pDb; + + assert( db!=0 ); + assert( iDb>=0 && iDbnDb ); + assert( zTabName ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ + pDb = &db->aDb[iDb]; + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, + sqlite3Strlen30(zTabName),0); + sqlite3DeleteTable(db, p); + db->flags |= SQLITE_InternChanges; } /* -** This function is responsible for invoking the collation factory callback -** or substituting a collation sequence of a different encoding when the -** requested collation sequence is not available in the desired encoding. -** -** If it is not NULL, then pColl must point to the database native encoding -** collation sequence with name zName, length nName. +** Given a token, return a string that consists of the text of that +** token. Space to hold the returned string +** is obtained from sqliteMalloc() and must be freed by the calling +** function. ** -** The return value is either the collation sequence to be used in database -** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. +** Any quotation marks (ex: "name", 'name', [name], or `name`) that +** surround the body of the token are removed. ** -** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() +** Tokens are often just pointers into the original SQL text and so +** are not \000 terminated and are not persistent. The returned string +** is \000 terminated and is persistent. */ -SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( - sqlite3* db, /* The database connection */ - u8 enc, /* The desired encoding for the collating sequence */ - CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ - const char *zName /* Collating sequence name */ -){ - CollSeq *p; - - p = pColl; - if( !p ){ - p = sqlite3FindCollSeq(db, enc, zName, 0); - } - if( !p || !p->xCmp ){ - /* No collation sequence of this type for this encoding is registered. - ** Call the collation factory to see if it can supply us with one. - */ - callCollNeeded(db, enc, zName); - p = sqlite3FindCollSeq(db, enc, zName, 0); - } - if( p && !p->xCmp && synthCollSeq(db, p) ){ - p = 0; +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ + char *zName; + if( pName ){ + zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n); + sqlite3Dequote(zName); + }else{ + zName = 0; } - assert( !p || p->xCmp ); - return p; + return zName; } /* -** This routine is called on a collation sequence before it is used to -** check that it is defined. An undefined collation sequence exists when -** a database is loaded that contains references to collation sequences -** that have not been defined by sqlite3_create_collation() etc. -** -** If required, this routine calls the 'collation needed' callback to -** request a definition of the collating sequence. If this doesn't work, -** an equivalent collating sequence that uses a text encoding different -** from the main database is substituted, if one is available. +** Open the sqlite_master table stored in database number iDb for +** writing. The table is opened using cursor 0. */ -SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ - if( pColl ){ - const char *zName = pColl->zName; - sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); - if( !p ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - pParse->nErr++; - return SQLITE_ERROR; - } - assert( p==pColl ); +SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *p, int iDb){ + Vdbe *v = sqlite3GetVdbe(p); + sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); + sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, MASTER_ROOT, iDb); + sqlite3VdbeChangeP4(v, -1, (char *)5, P4_INT32); /* 5 column table */ + if( p->nTab==0 ){ + p->nTab = 1; } - return SQLITE_OK; } - - /* -** Locate and return an entry from the db.aCollSeq hash table. If the entry -** specified by zName and nName is not found and parameter 'create' is -** true, then create a new entry. Otherwise return NULL. -** -** Each pointer stored in the sqlite3.aCollSeq hash table contains an -** array of three CollSeq structures. The first is the collation sequence -** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. -** -** Stored immediately after the three collation sequences is a copy of -** the collation sequence name. A pointer to this string is stored in -** each collation sequence structure. +** Parameter zName points to a nul-terminated buffer containing the name +** of a database ("main", "temp" or the name of an attached db). This +** function returns the index of the named database in db->aDb[], or +** -1 if the named db cannot be found. */ -static CollSeq *findCollSeqEntry( - sqlite3 *db, /* Database connection */ - const char *zName, /* Name of the collating sequence */ - int create /* Create a new entry if true */ -){ - CollSeq *pColl; - int nName = sqlite3Strlen30(zName); - pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); - - if( 0==pColl && create ){ - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); - if( pColl ){ - CollSeq *pDel = 0; - pColl[0].zName = (char*)&pColl[3]; - pColl[0].enc = SQLITE_UTF8; - pColl[1].zName = (char*)&pColl[3]; - pColl[1].enc = SQLITE_UTF16LE; - pColl[2].zName = (char*)&pColl[3]; - pColl[2].enc = SQLITE_UTF16BE; - memcpy(pColl[0].zName, zName, nName); - pColl[0].zName[nName] = 0; - pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); - - /* If a malloc() failure occurred in sqlite3HashInsert(), it will - ** return the pColl pointer to be deleted (because it wasn't added - ** to the hash table). - */ - assert( pDel==0 || pDel==pColl ); - if( pDel!=0 ){ - db->mallocFailed = 1; - sqlite3DbFree(db, pDel); - pColl = 0; +SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ + int i = -1; /* Database number */ + if( zName ){ + Db *pDb; + int n = sqlite3Strlen30(zName); + for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ + if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && + 0==sqlite3StrICmp(pDb->zName, zName) ){ + break; } } } - return pColl; + return i; } /* -** Parameter zName points to a UTF-8 encoded string nName bytes long. -** Return the CollSeq* pointer for the collation sequence named zName -** for the encoding 'enc' from the database 'db'. -** -** If the entry specified is not found and 'create' is true, then create a -** new entry. Otherwise return NULL. -** -** A separate function sqlite3LocateCollSeq() is a wrapper around -** this routine. sqlite3LocateCollSeq() invokes the collation factory -** if necessary and generates an error message if the collating sequence -** cannot be found. -** -** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() +** The token *pName contains the name of a database (either "main" or +** "temp" or the name of an attached db). This routine returns the +** index of the named database in db->aDb[], or -1 if the named db +** does not exist. */ -SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( - sqlite3 *db, - u8 enc, - const char *zName, - int create -){ - CollSeq *pColl; - if( zName ){ - pColl = findCollSeqEntry(db, zName, create); - }else{ - pColl = db->pDfltColl; - } - assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); - assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); - if( pColl ) pColl += enc-1; - return pColl; +SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){ + int i; /* Database number */ + char *zName; /* Name we are searching for */ + zName = sqlite3NameFromToken(db, pName); + i = sqlite3FindDbName(db, zName); + sqlite3DbFree(db, zName); + return i; } -/* During the search for the best function definition, this procedure -** is called to test how well the function passed as the first argument -** matches the request for a function with nArg arguments in a system -** that uses encoding enc. The value returned indicates how well the -** request is matched. A higher value indicates a better match. +/* The table or view or trigger name is passed to this routine via tokens +** pName1 and pName2. If the table name was fully qualified, for example: ** -** The returned value is always between 0 and 6, as follows: +** CREATE TABLE xxx.yyy (...); +** +** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if +** the table name is not fully qualified, i.e.: ** -** 0: Not a match, or if nArg<0 and the function is has no implementation. -** 1: A variable arguments function that prefers UTF-8 when a UTF-16 -** encoding is requested, or vice versa. -** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is -** requested, or vice versa. -** 3: A variable arguments function using the same text encoding. -** 4: A function with the exact number of arguments requested that -** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. -** 5: A function with the exact number of arguments requested that -** prefers UTF-16LE when UTF-16BE is requested, or vice versa. -** 6: An exact match. +** CREATE TABLE yyy(...); ** +** Then pName1 is set to "yyy" and pName2 is "". +** +** This routine sets the *ppUnqual pointer to point at the token (pName1 or +** pName2) that stores the unqualified table name. The index of the +** database "xxx" is returned. */ -static int matchQuality(FuncDef *p, int nArg, u8 enc){ - int match = 0; - if( p->nArg==-1 || p->nArg==nArg - || (nArg==-1 && (p->xFunc!=0 || p->xStep!=0)) - ){ - match = 1; - if( p->nArg==nArg || nArg==-1 ){ - match = 4; - } - if( enc==p->iPrefEnc ){ - match += 2; +SQLITE_PRIVATE int sqlite3TwoPartName( + Parse *pParse, /* Parsing and code generating context */ + Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */ + Token *pName2, /* The "yyy" in the name "xxx.yyy" */ + Token **pUnqual /* Write the unqualified object name here */ +){ + int iDb; /* Database holding the object */ + sqlite3 *db = pParse->db; + + if( ALWAYS(pName2!=0) && pName2->n>0 ){ + if( db->init.busy ) { + sqlite3ErrorMsg(pParse, "corrupt database"); + pParse->nErr++; + return -1; } - else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || - (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ - match += 1; + *pUnqual = pName2; + iDb = sqlite3FindDb(db, pName1); + if( iDb<0 ){ + sqlite3ErrorMsg(pParse, "unknown database %T", pName1); + pParse->nErr++; + return -1; } + }else{ + assert( db->init.iDb==0 || db->init.busy ); + iDb = db->init.iDb; + *pUnqual = pName1; } - return match; + return iDb; } /* -** Search a FuncDefHash for a function with the given name. Return -** a pointer to the matching FuncDef if found, or 0 if there is no match. +** This routine is used to check if the UTF-8 string zName is a legal +** unqualified name for a new schema object (table, index, view or +** trigger). All names are legal except those that begin with the string +** "sqlite_" (in upper, lower or mixed case). This portion of the namespace +** is reserved for internal use. */ -static FuncDef *functionSearch( - FuncDefHash *pHash, /* Hash table to search */ - int h, /* Hash of the name */ - const char *zFunc, /* Name of function */ - int nFunc /* Number of bytes in zFunc */ -){ - FuncDef *p; - for(p=pHash->a[h]; p; p=p->pHash){ - if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ - return p; - } - } - return 0; -} - -/* -** Insert a new FuncDef into a FuncDefHash hash table. -*/ -SQLITE_PRIVATE void sqlite3FuncDefInsert( - FuncDefHash *pHash, /* The hash table into which to insert */ - FuncDef *pDef /* The function definition to insert */ -){ - FuncDef *pOther; - int nName = sqlite3Strlen30(pDef->zName); - u8 c1 = (u8)pDef->zName[0]; - int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); - pOther = functionSearch(pHash, h, pDef->zName, nName); - if( pOther ){ - assert( pOther!=pDef && pOther->pNext!=pDef ); - pDef->pNext = pOther->pNext; - pOther->pNext = pDef; - }else{ - pDef->pNext = 0; - pDef->pHash = pHash->a[h]; - pHash->a[h] = pDef; +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ + if( !pParse->db->init.busy && pParse->nested==0 + && (pParse->db->flags & SQLITE_WriteSchema)==0 + && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", zName); + return SQLITE_ERROR; } + return SQLITE_OK; } - - /* -** Locate a user function given a name, a number of arguments and a flag -** indicating whether the function prefers UTF-16 over UTF-8. Return a -** pointer to the FuncDef structure that defines that function, or return -** NULL if the function does not exist. -** -** If the createFlag argument is true, then a new (blank) FuncDef -** structure is created and liked into the "db" structure if a -** no matching function previously existed. When createFlag is true -** and the nArg parameter is -1, then only a function that accepts -** any number of arguments will be returned. -** -** If createFlag is false and nArg is -1, then the first valid -** function found is returned. A function is valid if either xFunc -** or xStep is non-zero. +** Begin constructing a new table representation in memory. This is +** the first of several action routines that get called in response +** to a CREATE TABLE statement. In particular, this routine is called +** after seeing tokens "CREATE" and "TABLE" and the table name. The isTemp +** flag is true if the table should be stored in the auxiliary database +** file instead of in the main database file. This is normally the case +** when the "TEMP" or "TEMPORARY" keyword occurs in between +** CREATE and TABLE. ** -** If createFlag is false, then a function with the required name and -** number of arguments may be returned even if the eTextRep flag does not -** match that requested. +** The new table record is initialized and put in pParse->pNewTable. +** As more of the CREATE TABLE statement is parsed, additional action +** routines will be called to add more information to this record. +** At the end of the CREATE TABLE statement, the sqlite3EndTable() routine +** is called to complete the construction of the new table record. */ -SQLITE_PRIVATE FuncDef *sqlite3FindFunction( - sqlite3 *db, /* An open database */ - const char *zName, /* Name of the function. Not null-terminated */ - int nName, /* Number of characters in the name */ - int nArg, /* Number of arguments. -1 means any number */ - u8 enc, /* Preferred text encoding */ - int createFlag /* Create new entry if true and does not otherwise exist */ +SQLITE_PRIVATE void sqlite3StartTable( + Parse *pParse, /* Parser context */ + Token *pName1, /* First part of the name of the table or view */ + Token *pName2, /* Second part of the name of the table or view */ + int isTemp, /* True if this is a TEMP table */ + int isView, /* True if this is a VIEW */ + int isVirtual, /* True if this is a VIRTUAL table */ + int noErr /* Do nothing if table already exists */ ){ - FuncDef *p; /* Iterator variable */ - FuncDef *pBest = 0; /* Best match found so far */ - int bestScore = 0; /* Score of best match */ - int h; /* Hash value */ - - - assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); + Table *pTable; + char *zName = 0; /* The name of the new table */ + sqlite3 *db = pParse->db; + Vdbe *v; + int iDb; /* Database number to create the table in */ + Token *pName; /* Unqualified name of the table to create */ - /* First search for a match amongst the application-defined functions. + /* The table or view name to create is passed to this routine via tokens + ** pName1 and pName2. If the table name was fully qualified, for example: + ** + ** CREATE TABLE xxx.yyy (...); + ** + ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if + ** the table name is not fully qualified, i.e.: + ** + ** CREATE TABLE yyy(...); + ** + ** Then pName1 is set to "yyy" and pName2 is "". + ** + ** The call below sets the pName pointer to point at the token (pName1 or + ** pName2) that stores the unqualified table name. The variable iDb is + ** set to the index of the database that the table or view is to be + ** created in. */ - p = functionSearch(&db->aFunc, h, zName, nName); - while( p ){ - int score = matchQuality(p, nArg, enc); - if( score>bestScore ){ - pBest = p; - bestScore = score; + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + + pParse->sNameToken = *pName; + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ) return; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + goto begin_table_error; + } + if( db->init.iDb==1 ) isTemp = 1; +#ifndef SQLITE_OMIT_AUTHORIZATION + assert( (isTemp & 1)==isTemp ); + { + int code; + char *zDb = db->aDb[iDb].zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ + goto begin_table_error; + } + if( isView ){ + if( !OMIT_TEMPDB && isTemp ){ + code = SQLITE_CREATE_TEMP_VIEW; + }else{ + code = SQLITE_CREATE_VIEW; + } + }else{ + if( !OMIT_TEMPDB && isTemp ){ + code = SQLITE_CREATE_TEMP_TABLE; + }else{ + code = SQLITE_CREATE_TABLE; + } + } + if( !isVirtual && sqlite3AuthCheck(pParse, code, zName, 0, zDb) ){ + goto begin_table_error; } - p = p->pNext; } +#endif - /* If no match is found, search the built-in functions. - ** - ** If the SQLITE_PreferBuiltin flag is set, then search the built-in - ** functions even if a prior app-defined function was found. And give - ** priority to built-in functions. - ** - ** Except, if createFlag is true, that means that we are trying to - ** install a new function. Whatever FuncDef structure is returned it will - ** have fields overwritten with new information appropriate for the - ** new function. But the FuncDefs for built-in functions are read-only. - ** So we must not search for built-ins when creating a new function. - */ - if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - bestScore = 0; - p = functionSearch(pHash, h, zName, nName); - while( p ){ - int score = matchQuality(p, nArg, enc); - if( score>bestScore ){ - pBest = p; - bestScore = score; + /* Make sure the new table name does not collide with an existing + ** index or table name in the same database. Issue an error message if + ** it does. The exception is if the statement being parsed was passed + ** to an sqlite3_declare_vtab() call. In that case only the column names + ** and types will be used, so there is no need to test for namespace + ** collisions. + */ + if( !IN_DECLARE_VTAB ){ + char *zDb = db->aDb[iDb].zName; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto begin_table_error; + } + pTable = sqlite3FindTable(db, zName, zDb); + if( pTable ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "table %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); } - p = p->pNext; + goto begin_table_error; + } + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); + goto begin_table_error; } } - /* If the createFlag parameter is true and the search did not reveal an - ** exact match for the name, number of arguments and encoding, then add a - ** new entry to the hash table and return it. + pTable = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTable==0 ){ + db->mallocFailed = 1; + pParse->rc = SQLITE_NOMEM; + pParse->nErr++; + goto begin_table_error; + } + pTable->zName = zName; + pTable->iPKey = -1; + pTable->pSchema = db->aDb[iDb].pSchema; + pTable->nRef = 1; + pTable->nRowEst = 1000000; + assert( pParse->pNewTable==0 ); + pParse->pNewTable = pTable; + + /* If this is the magic sqlite_sequence table used by autoincrement, + ** then record a pointer to this table in the main database structure + ** so that INSERT can find the table easily. */ - if( createFlag && (bestScore<6 || pBest->nArg!=nArg) && - (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ - pBest->zName = (char *)&pBest[1]; - pBest->nArg = (u16)nArg; - pBest->iPrefEnc = enc; - memcpy(pBest->zName, zName, nName); - pBest->zName[nName] = 0; - sqlite3FuncDefInsert(&db->aFunc, pBest); +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( !pParse->nested && strcmp(zName, "sqlite_sequence")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pTable->pSchema->pSeqTab = pTable; } +#endif - if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ - return pBest; + /* Begin generating the code that will insert the table record into + ** the SQLITE_MASTER table. Note in particular that we must go ahead + ** and allocate the record number for the table entry now. Before any + ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause + ** indices to be created and the table record must come before the + ** indices. Hence, the record number for the table must be allocated + ** now. + */ + if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ + int j1; + int fileFormat; + int reg1, reg2, reg3; + sqlite3BeginWriteOperation(pParse, 0, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( isVirtual ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* If the file format and encoding in the database have not been set, + ** set them now. + */ + reg1 = pParse->regRowid = ++pParse->nMem; + reg2 = pParse->regRoot = ++pParse->nMem; + reg3 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); + fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? + 1 : SQLITE_MAX_FILE_FORMAT; + sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); + sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); + sqlite3VdbeJumpHere(v, j1); + + /* This just creates a place-holder record in the sqlite_master table. + ** The record created does not contain anything yet. It will be replaced + ** by the real entry in code generated at sqlite3EndTable(). + ** + ** The rowid for the new entry is left in register pParse->regRowid. + ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid and root page number values are needed by the code that + ** sqlite3EndTable will generate. + */ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( isView || isVirtual ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); + }else +#endif + { + sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); + } + sqlite3OpenMasterTable(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); + sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); } - return 0; + + /* Normal (non-error) return. */ + return; + + /* If an error occurs, we jump here */ +begin_table_error: + sqlite3DbFree(db, zName); + return; } /* -** Free all resources held by the schema structure. The void* argument points -** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the -** pointer itself, it just cleans up subsiduary resources (i.e. the contents -** of the schema hash tables). +** This macro is used to compare two strings in a case-insensitive manner. +** It is slightly faster than calling sqlite3StrICmp() directly, but +** produces larger code. ** -** The Schema.cache_size variable is not cleared. +** WARNING: This macro is not compatible with the strcmp() family. It +** returns true if the two strings are equal, otherwise false. */ -SQLITE_PRIVATE void sqlite3SchemaFree(void *p){ - Hash temp1; - Hash temp2; - HashElem *pElem; - Schema *pSchema = (Schema *)p; +#define STRICMP(x, y) (\ +sqlite3UpperToLower[*(unsigned char *)(x)]== \ +sqlite3UpperToLower[*(unsigned char *)(y)] \ +&& sqlite3StrICmp((x)+1,(y)+1)==0 ) - temp1 = pSchema->tblHash; - temp2 = pSchema->trigHash; - sqlite3HashInit(&pSchema->trigHash); - sqlite3HashClear(&pSchema->idxHash); - for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); +/* +** Add a new column to the table currently being constructed. +** +** The parser calls this routine once for each column declaration +** in a CREATE TABLE statement. sqlite3StartTable() gets called +** first to get things going. Then this routine is called for each +** column. +*/ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ + Table *p; + int i; + char *z; + Column *pCol; + sqlite3 *db = pParse->db; + if( (p = pParse->pNewTable)==0 ) return; +#if SQLITE_MAX_COLUMN + if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); + return; } - sqlite3HashClear(&temp2); - sqlite3HashInit(&pSchema->tblHash); - for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ - Table *pTab = sqliteHashData(pElem); - sqlite3DeleteTable(0, pTab); +#endif + z = sqlite3NameFromToken(db, pName); + if( z==0 ) return; + for(i=0; inCol; i++){ + if( STRICMP(z, p->aCol[i].zName) ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; + } } - sqlite3HashClear(&temp1); - sqlite3HashClear(&pSchema->fkeyHash); - pSchema->pSeqTab = 0; - pSchema->flags &= ~DB_SchemaLoaded; + if( (p->nCol & 0x7)==0 ){ + Column *aNew; + aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0])); + if( aNew==0 ){ + sqlite3DbFree(db, z); + return; + } + p->aCol = aNew; + } + pCol = &p->aCol[p->nCol]; + memset(pCol, 0, sizeof(p->aCol[0])); + pCol->zName = z; + + /* If there is no type specified, columns have the default affinity + ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will + ** be called next to set pCol->affinity correctly. + */ + pCol->affinity = SQLITE_AFF_NONE; + p->nCol++; } /* -** Find and return the schema associated with a BTree. Create -** a new one if necessary. +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. A "NOT NULL" constraint has +** been seen on a column. This routine sets the notNull flag on +** the column currently under construction. */ -SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ - Schema * p; - if( pBt ){ - p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaFree); - }else{ - p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); - } - if( !p ){ - db->mallocFailed = 1; - }else if ( 0==p->file_format ){ - sqlite3HashInit(&p->tblHash); - sqlite3HashInit(&p->idxHash); - sqlite3HashInit(&p->trigHash); - sqlite3HashInit(&p->fkeyHash); - p->enc = SQLITE_UTF8; - } - return p; +SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ + Table *p; + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + p->aCol[p->nCol-1].notNull = (u8)onError; } -/************** End of callback.c ********************************************/ -/************** Begin file delete.c ******************************************/ /* -** 2001 September 15 +** Scan the column type name zType (length nType) and return the +** associated affinity type. ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** This routine does a case-independent search of zType for the +** substrings in the following table. If one of the substrings is +** found, the corresponding affinity is returned. If zType contains +** more than one of the substrings, entries toward the top of +** the table take priority. For example, if zType is 'BLOBINT', +** SQLITE_AFF_INTEGER is returned. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Substring | Affinity +** -------------------------------- +** 'INT' | SQLITE_AFF_INTEGER +** 'CHAR' | SQLITE_AFF_TEXT +** 'CLOB' | SQLITE_AFF_TEXT +** 'TEXT' | SQLITE_AFF_TEXT +** 'BLOB' | SQLITE_AFF_NONE +** 'REAL' | SQLITE_AFF_REAL +** 'FLOA' | SQLITE_AFF_REAL +** 'DOUB' | SQLITE_AFF_REAL ** -************************************************************************* -** This file contains C code routines that are called by the parser -** in order to generate code for DELETE FROM statements. +** If none of the substrings in the above table are found, +** SQLITE_AFF_NUMERIC is returned. */ +SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn){ + u32 h = 0; + char aff = SQLITE_AFF_NUMERIC; -/* -** Look up every table that is named in pSrc. If any table is not found, -** add an error message to pParse->zErrMsg and return NULL. If all tables -** are found, return a pointer to the last table. -*/ -SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ - struct SrcList_item *pItem = pSrc->a; - Table *pTab; - assert( pItem && pSrc->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); - sqlite3DeleteTable(pParse->db, pItem->pTab); - pItem->pTab = pTab; - if( pTab ){ - pTab->nRef++; - } - if( sqlite3IndexedByLookup(pParse, pItem) ){ - pTab = 0; + if( zIn ) while( zIn[0] ){ + h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + zIn++; + if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ + && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ + aff = SQLITE_AFF_NONE; +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('f'<<24)+('l'<<16)+('o'<<8)+'a') /* FLOA */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('d'<<24)+('o'<<16)+('u'<<8)+'b') /* DOUB */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; +#endif + }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){ /* INT */ + aff = SQLITE_AFF_INTEGER; + break; + } } - return pTab; + + return aff; } /* -** Check to make sure the given table is writable. If it is not -** writable, generate an error message and return 1. If it is -** writable return 0; -*/ -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ - /* A table is not writable under the following circumstances: - ** - ** 1) It is a virtual table and no implementation of the xUpdate method - ** has been provided, or - ** 2) It is a system table (i.e. sqlite_master), this call is not - ** part of a nested parse and writable_schema pragma has not - ** been specified. - ** - ** In either case leave an error message in pParse and return non-zero. - */ - if( ( IsVirtual(pTab) - && sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ) - || ( (pTab->tabFlags & TF_Readonly)!=0 - && (pParse->db->flags & SQLITE_WriteSchema)==0 - && pParse->nested==0 ) - ){ - sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); - return 1; - } - -#ifndef SQLITE_OMIT_VIEW - if( !viewOk && pTab->pSelect ){ - sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); - return 1; - } -#endif - return 0; -} +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. The pFirst token is the first +** token in the sequence of tokens that describe the type of the +** column currently under construction. pLast is the last token +** in the sequence. Use this information to construct a string +** that contains the typename of the column and store that string +** in zType. +*/ +SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){ + Table *p; + Column *pCol; + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + pCol = &p->aCol[p->nCol-1]; + assert( pCol->zType==0 ); + pCol->zType = sqlite3NameFromToken(pParse->db, pType); + pCol->affinity = sqlite3AffinityType(pCol->zType); +} -#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) /* -** Evaluate a view and store its result in an ephemeral table. The -** pWhere argument is an optional WHERE clause that restricts the -** set of rows in the view that are to be added to the ephemeral table. +** The expression is the default value for the most recently added column +** of the table currently under construction. +** +** Default value expressions must be constant. Raise an exception if this +** is not the case. +** +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. */ -SQLITE_PRIVATE void sqlite3MaterializeView( - Parse *pParse, /* Parsing context */ - Table *pView, /* View definition */ - Expr *pWhere, /* Optional WHERE clause to be added */ - int iCur /* Cursor number for ephemerial table */ -){ - SelectDest dest; - Select *pDup; +SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ + Table *p; + Column *pCol; sqlite3 *db = pParse->db; - - pDup = sqlite3SelectDup(db, pView->pSelect, 0); - if( pWhere ){ - SrcList *pFrom; - - pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ - assert( pFrom->nSrc==1 ); - pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].pSelect = pDup; - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); + p = pParse->pNewTable; + if( p!=0 ){ + pCol = &(p->aCol[p->nCol-1]); + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", + pCol->zName); }else{ - sqlite3SelectDelete(db, pDup); + /* A copy of pExpr is used instead of the original, as pExpr contains + ** tokens that point to volatile memory. The 'span' of the expression + ** is required by pragma table_info. + */ + sqlite3ExprDelete(db, pCol->pDflt); + pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); + sqlite3DbFree(db, pCol->zDflt); + pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); } - pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); } - sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); - sqlite3Select(pParse, pDup, &dest); - sqlite3SelectDelete(db, pDup); + sqlite3ExprDelete(db, pSpan->pExpr); } -#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ -#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) /* -** Generate an expression tree to implement the WHERE, ORDER BY, -** and LIMIT/OFFSET portion of DELETE and UPDATE statements. +** Designate the PRIMARY KEY for the table. pList is a list of names +** of columns that form the primary key. If pList is NULL, then the +** most recently added column of the table is the primary key. ** -** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; -** \__________________________/ -** pLimitWhere (pInClause) +** A table can have at most one primary key. If the table already has +** a primary key (and this is the second primary key) then create an +** error. +** +** If the PRIMARY KEY is on a single column whose datatype is INTEGER, +** then we will try to use that column as the rowid. Set the Table.iPKey +** field of the table under construction to be the index of the +** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is +** no INTEGER PRIMARY KEY. +** +** If the key is not an INTEGER PRIMARY KEY, then create a unique +** index for the key. No index is created for INTEGER PRIMARY KEYs. */ -SQLITE_PRIVATE Expr *sqlite3LimitWhere( - Parse *pParse, /* The parser context */ - SrcList *pSrc, /* the FROM clause -- which tables to scan */ - Expr *pWhere, /* The WHERE clause. May be null */ - ExprList *pOrderBy, /* The ORDER BY clause. May be null */ - Expr *pLimit, /* The LIMIT clause. May be null */ - Expr *pOffset, /* The OFFSET clause. May be null */ - char *zStmtType /* Either DELETE or UPDATE. For error messages. */ +SQLITE_PRIVATE void sqlite3AddPrimaryKey( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List of field names to be indexed */ + int onError, /* What to do with a uniqueness conflict */ + int autoInc, /* True if the AUTOINCREMENT keyword is present */ + int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ ){ - Expr *pWhereRowid = NULL; /* WHERE rowid .. */ - Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - Expr *pSelectRowid = NULL; /* SELECT rowid ... */ - ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ - SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ - Select *pSelect = NULL; /* Complete SELECT tree */ - - /* Check that there isn't an ORDER BY without a LIMIT clause. - */ - if( pOrderBy && (pLimit == 0) ) { - sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - pParse->parseError = 1; - goto limit_where_cleanup_2; + Table *pTab = pParse->pNewTable; + char *zType = 0; + int iCol = -1, i; + if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; + if( pTab->tabFlags & TF_HasPrimaryKey ){ + sqlite3ErrorMsg(pParse, + "table \"%s\" has more than one primary key", pTab->zName); + goto primary_key_exit; } - - /* We only need to generate a select expression if there - ** is a limit/offset term to enforce. - */ - if( pLimit == 0 ) { - /* if pLimit is null, pOffset will always be null as well. */ - assert( pOffset == 0 ); - return pWhere; + pTab->tabFlags |= TF_HasPrimaryKey; + if( pList==0 ){ + iCol = pTab->nCol - 1; + pTab->aCol[iCol].isPrimKey = 1; + }else{ + for(i=0; inExpr; i++){ + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ + break; + } + } + if( iColnCol ){ + pTab->aCol[iCol].isPrimKey = 1; + } + } + if( pList->nExpr>1 ) iCol = -1; } - - /* Generate a select expression tree to enforce the limit/offset - ** term for the DELETE or UPDATE statement. For example: - ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 - ** becomes: - ** DELETE FROM table_a WHERE rowid IN ( - ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 - ** ); - */ - - pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup_2; - pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup_2; - - /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree - ** and the SELECT subtree. */ - pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); - if( pSelectSrc == 0 ) { - sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup_2; + if( iCol>=0 && iColnCol ){ + zType = pTab->aCol[iCol].zType; + } + if( zType && sqlite3StrICmp(zType, "INTEGER")==0 + && sortOrder==SQLITE_SO_ASC ){ + pTab->iPKey = iCol; + pTab->keyConf = (u8)onError; + assert( autoInc==0 || autoInc==1 ); + pTab->tabFlags |= autoInc*TF_Autoincrement; + }else if( autoInc ){ +#ifndef SQLITE_OMIT_AUTOINCREMENT + sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " + "INTEGER PRIMARY KEY"); +#endif + }else{ + Index *p; + p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0); + if( p ){ + p->autoIndex = 2; + } + pList = 0; } - /* generate the SELECT expression tree. */ - pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0, - pOrderBy,0,pLimit,pOffset); - if( pSelect == 0 ) return 0; +primary_key_exit: + sqlite3ExprListDelete(pParse->db, pList); + return; +} - /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ - pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pWhereRowid == 0 ) goto limit_where_cleanup_1; - pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); - if( pInClause == 0 ) goto limit_where_cleanup_1; +/* +** Add a new CHECK constraint to the table currently under construction. +*/ +SQLITE_PRIVATE void sqlite3AddCheckConstraint( + Parse *pParse, /* Parsing context */ + Expr *pCheckExpr /* The check expression */ +){ + sqlite3 *db = pParse->db; +#ifndef SQLITE_OMIT_CHECK + Table *pTab = pParse->pNewTable; + if( pTab && !IN_DECLARE_VTAB ){ + pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); + }else +#endif + { + sqlite3ExprDelete(db, pCheckExpr); + } +} - pInClause->x.pSelect = pSelect; - pInClause->flags |= EP_xIsSelect; - sqlite3ExprSetHeight(pParse, pInClause); - return pInClause; +/* +** Set the collation function of the most recently parsed table column +** to the CollSeq given. +*/ +SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ + Table *p; + int i; + char *zColl; /* Dequoted name of collation sequence */ + sqlite3 *db; - /* something went wrong. clean up anything allocated. */ -limit_where_cleanup_1: - sqlite3SelectDelete(pParse->db, pSelect); - return 0; + if( (p = pParse->pNewTable)==0 ) return; + i = p->nCol-1; + db = pParse->db; + zColl = sqlite3NameFromToken(db, pToken); + if( !zColl ) return; -limit_where_cleanup_2: - sqlite3ExprDelete(pParse->db, pWhere); - sqlite3ExprListDelete(pParse->db, pOrderBy); - sqlite3ExprDelete(pParse->db, pLimit); - sqlite3ExprDelete(pParse->db, pOffset); - return 0; + if( sqlite3LocateCollSeq(pParse, zColl) ){ + Index *pIdx; + p->aCol[i].zColl = zColl; + + /* If the column is declared as " PRIMARY KEY COLLATE ", + ** then an index may have been created on this column before the + ** collation type was added. Correct this if it is the case. + */ + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nColumn==1 ); + if( pIdx->aiColumn[0]==i ){ + pIdx->azColl[0] = p->aCol[i].zColl; + } + } + }else{ + sqlite3DbFree(db, zColl); + } } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ /* -** Generate code for a DELETE FROM statement. +** This function returns the collation sequence for database native text +** encoding identified by the string zName, length nName. ** -** DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL; -** \________/ \________________/ -** pTabList pWhere +** If the requested collation sequence is not available, or not available +** in the database native encoding, the collation factory is invoked to +** request it. If the collation factory does not supply such a sequence, +** and the sequence is available in another text encoding, then that is +** returned instead. +** +** If no versions of the requested collations sequence are available, or +** another error occurs, NULL is returned and an error message written into +** pParse. +** +** This routine is a wrapper around sqlite3FindCollSeq(). This routine +** invokes the collation factory if the named collation cannot be found +** and generates an error message. +** +** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() */ -SQLITE_PRIVATE void sqlite3DeleteFrom( - Parse *pParse, /* The parser context */ - SrcList *pTabList, /* The table from which we should delete things */ - Expr *pWhere /* The WHERE clause. May be null */ -){ - Vdbe *v; /* The virtual database engine */ - Table *pTab; /* The table from which records will be deleted */ - const char *zDb; /* Name of database holding pTab */ - int end, addr = 0; /* A couple addresses of generated code */ - int i; /* Loop counter */ - WhereInfo *pWInfo; /* Information about the WHERE clause */ - Index *pIdx; /* For looping over indices of the table */ - int iCur; /* VDBE Cursor number for pTab */ - sqlite3 *db; /* Main database structure */ - AuthContext sContext; /* Authorization context */ - NameContext sNC; /* Name context to resolve expressions in */ - int iDb; /* Database number */ - int memCnt = -1; /* Memory cell used for change counting */ - int rcauth; /* Value returned by authorization callback */ - -#ifndef SQLITE_OMIT_TRIGGER - int isView; /* True if attempting to delete from a view */ - Trigger *pTrigger; /* List of table triggers, if required */ -#endif +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ + sqlite3 *db = pParse->db; + u8 enc = ENC(db); + u8 initbusy = db->init.busy; + CollSeq *pColl; - memset(&sContext, 0, sizeof(sContext)); - db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ - goto delete_from_cleanup; + pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); + if( !initbusy && (!pColl || !pColl->xCmp) ){ + pColl = sqlite3GetCollSeq(db, enc, pColl, zName); + if( !pColl ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + } } - assert( pTabList->nSrc==1 ); - /* Locate the table which we want to delete. This table has to be - ** put in an SrcList structure because some of the subroutines we - ** will be calling are designed to work with multiple tables and expect - ** an SrcList* parameter instead of just a Table* parameter. - */ - pTab = sqlite3SrcListLookup(pParse, pTabList); - if( pTab==0 ) goto delete_from_cleanup; + return pColl; +} - /* Figure out if we have any triggers and if the table being - ** deleted from is a view - */ -#ifndef SQLITE_OMIT_TRIGGER - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - isView = pTab->pSelect!=0; -#else -# define pTrigger 0 -# define isView 0 -#endif -#ifdef SQLITE_OMIT_VIEW -# undef isView -# define isView 0 -#endif - /* If pTab is really a view, make sure it has been initialized. - */ - if( sqlite3ViewGetColumnNames(pParse, pTab) ){ - goto delete_from_cleanup; - } +/* +** Generate code that will increment the schema cookie. +** +** The schema cookie is used to determine when the schema for the +** database changes. After each schema change, the cookie value +** changes. When a process first reads the schema it records the +** cookie. Thereafter, whenever it goes to access the database, +** it checks the cookie to make sure the schema has not changed +** since it was last read. +** +** This plan is not completely bullet-proof. It is possible for +** the schema to change multiple times and for the cookie to be +** set back to prior value. But schema changes are infrequent +** and the probability of hitting the same cookie value is only +** 1 chance in 2^32. So we're safe enough. +*/ +SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, r1); + sqlite3ReleaseTempReg(pParse, r1); +} - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ - goto delete_from_cleanup; - } - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( iDbnDb ); - zDb = db->aDb[iDb].zName; - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); - assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); - if( rcauth==SQLITE_DENY ){ - goto delete_from_cleanup; +/* +** Measure the number of characters needed to output the given +** identifier. The number returned includes any quotes used +** but does not include the null terminator. +** +** The estimate is conservative. It might be larger that what is +** really needed. +*/ +static int identLength(const char *z){ + int n; + for(n=0; *z; n++, z++){ + if( *z=='"' ){ n++; } } - assert(!isView || pTrigger); + return n + 2; +} - /* Assign cursor number to the table and all its indices. - */ - assert( pTabList->nSrc==1 ); - iCur = pTabList->a[0].iCursor = pParse->nTab++; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - pParse->nTab++; - } +/* +** The first parameter is a pointer to an output buffer. The second +** parameter is a pointer to an integer that contains the offset at +** which to write into the output buffer. This function copies the +** nul-terminated string pointed to by the third parameter, zSignedIdent, +** to the specified offset in the buffer and updates *pIdx to refer +** to the first byte after the last byte written before returning. +** +** If the string zSignedIdent consists entirely of alpha-numeric +** characters, does not begin with a digit and is not an SQL keyword, +** then it is copied to the output buffer exactly as it is. Otherwise, +** it is quoted using double-quotes. +*/ +static void identPut(char *z, int *pIdx, char *zSignedIdent){ + unsigned char *zIdent = (unsigned char*)zSignedIdent; + int i, j, needQuote; + i = *pIdx; - /* Start the view context - */ - if( isView ){ - sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + for(j=0; zIdent[j]; j++){ + if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; } - - /* Begin generating code. - */ - v = sqlite3GetVdbe(pParse); - if( v==0 ){ - goto delete_from_cleanup; + needQuote = sqlite3Isdigit(zIdent[0]) || sqlite3KeywordCode(zIdent, j)!=TK_ID; + if( !needQuote ){ + needQuote = zIdent[j]; } - if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); - sqlite3BeginWriteOperation(pParse, 1, iDb); - /* If we are trying to delete from a view, realize that view into - ** a ephemeral table. - */ -#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) - if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iCur); + if( needQuote ) z[i++] = '"'; + for(j=0; zIdent[j]; j++){ + z[i++] = zIdent[j]; + if( zIdent[j]=='"' ) z[i++] = '"'; } -#endif + if( needQuote ) z[i++] = '"'; + z[i] = 0; + *pIdx = i; +} - /* Resolve the column names in the WHERE clause. - */ - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - sNC.pSrcList = pTabList; - if( sqlite3ResolveExprNames(&sNC, pWhere) ){ - goto delete_from_cleanup; +/* +** Generate a CREATE TABLE statement appropriate for the given +** table. Memory to hold the text of the statement is obtained +** from sqliteMalloc() and must be freed by the calling function. +*/ +static char *createTableStmt(sqlite3 *db, Table *p){ + int i, k, n; + char *zStmt; + char *zSep, *zSep2, *zEnd; + Column *pCol; + n = 0; + for(pCol = p->aCol, i=0; inCol; i++, pCol++){ + n += identLength(pCol->zName) + 5; } - - /* Initialize the counter of the number of rows deleted, if - ** we are counting rows. - */ - if( db->flags & SQLITE_CountRows ){ - memCnt = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); + n += identLength(p->zName); + if( n<50 ){ + zSep = ""; + zSep2 = ","; + zEnd = ")"; + }else{ + zSep = "\n "; + zSep2 = ",\n "; + zEnd = "\n)"; } + n += 35 + 6*p->nCol; + zStmt = sqlite3DbMallocRaw(0, n); + if( zStmt==0 ){ + db->mallocFailed = 1; + return 0; + } + sqlite3_snprintf(n, zStmt, "CREATE TABLE "); + k = sqlite3Strlen30(zStmt); + identPut(zStmt, &k, p->zName); + zStmt[k++] = '('; + for(pCol=p->aCol, i=0; inCol; i++, pCol++){ + static const char * const azType[] = { + /* SQLITE_AFF_TEXT */ " TEXT", + /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_NUMERIC */ " NUM", + /* SQLITE_AFF_INTEGER */ " INT", + /* SQLITE_AFF_REAL */ " REAL" + }; + int len; + const char *zType; -#ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION - /* Special case: A DELETE without a WHERE clause deletes everything. - ** It is easier just to erase the whole table. Prior to version 3.6.5, - ** this optimization caused the row change count (the value returned by - ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) - ){ - assert( !isView ); - sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt, - pTab->zName, P4_STATIC); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); - } - }else -#endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ - /* The usual case: There is a WHERE clause so we have to scan through - ** the table and pick which records to delete. - */ - { - int iRowSet = ++pParse->nMem; /* Register for rowset of rows to delete */ - int iRowid = ++pParse->nMem; /* Used for storing rowid values. */ - int regRowid; /* Actual register containing rowids */ - - /* Collect rowids of every row to be deleted. - */ - sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); - if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); - if( db->flags & SQLITE_CountRows ){ - sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); - } - sqlite3WhereEnd(pWInfo); + sqlite3_snprintf(n-k, &zStmt[k], zSep); + k += sqlite3Strlen30(&zStmt[k]); + zSep = zSep2; + identPut(zStmt, &k, pCol->zName); + assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); + assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); + testcase( pCol->affinity==SQLITE_AFF_NONE ); + testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); + testcase( pCol->affinity==SQLITE_AFF_INTEGER ); + testcase( pCol->affinity==SQLITE_AFF_REAL ); + + zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; + len = sqlite3Strlen30(zType); + assert( pCol->affinity==SQLITE_AFF_NONE + || pCol->affinity==sqlite3AffinityType(zType) ); + memcpy(&zStmt[k], zType, len); + k += len; + assert( k<=n ); + } + sqlite3_snprintf(n-k, &zStmt[k], "%s", zEnd); + return zStmt; +} - /* Delete every item whose key was written to the list during the - ** database scan. We have to delete items after the scan is complete - ** because deleting an item can change the scan order. */ - end = sqlite3VdbeMakeLabel(v); +/* +** This routine is called to report the final ")" that terminates +** a CREATE TABLE statement. +** +** The table structure that other action routines have been building +** is added to the internal hash tables, assuming no errors have +** occurred. +** +** An entry for the table is made in the master table on disk, unless +** this is a temporary table or db->init.busy==1. When db->init.busy==1 +** it means we are reading the sqlite_master table because we just +** connected to the database or because the sqlite_master table has +** recently changed, so the entry for this table already exists in +** the sqlite_master table. We do not want to create it again. +** +** If the pSelect argument is not NULL, it means that this routine +** was called to create a table generated from a +** "CREATE TABLE ... AS SELECT ..." statement. The column names of +** the new table will match the result set of the SELECT. +*/ +SQLITE_PRIVATE void sqlite3EndTable( + Parse *pParse, /* Parse context */ + Token *pCons, /* The ',' token after the last column defn. */ + Token *pEnd, /* The final ')' token in the CREATE TABLE */ + Select *pSelect /* Select from a "CREATE ... AS SELECT" */ +){ + Table *p; + sqlite3 *db = pParse->db; + int iDb; - /* Unless this is a view, open cursors for the table we are - ** deleting from and all its indices. If this is a view, then the - ** only effect this statement has is to fire the INSTEAD OF - ** triggers. */ - if( !isView ){ - sqlite3OpenTableAndIndices(pParse, pTab, iCur, OP_OpenWrite); - } + if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + return; + } + p = pParse->pNewTable; + if( p==0 ) return; - addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, end, iRowid); + assert( !db->init.busy || !pSelect ); - /* Delete the row */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); - sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVTab, P4_VTAB); - sqlite3MayAbort(pParse); - }else -#endif - { - int count = (pParse->nested==0); /* True to count changes */ - sqlite3GenerateRowDelete(pParse, pTab, iCur, iRowid, count, pTrigger, OE_Default); - } + iDb = sqlite3SchemaToIndex(db, p->pSchema); - /* End of the delete loop */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); - sqlite3VdbeResolveLabel(v, end); +#ifndef SQLITE_OMIT_CHECK + /* Resolve names in all CHECK constraint expressions. + */ + if( p->pCheck ){ + SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + NameContext sNC; /* Name context for pParse->pNewTable */ - /* Close the cursors open on the table and its indexes. */ - if( !isView && !IsVirtual(pTab) ){ - for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Close, iCur + i, pIdx->tnum); - } - sqlite3VdbeAddOp1(v, OP_Close, iCur); + memset(&sNC, 0, sizeof(sNC)); + memset(&sSrc, 0, sizeof(sSrc)); + sSrc.nSrc = 1; + sSrc.a[0].zName = p->zName; + sSrc.a[0].pTab = p; + sSrc.a[0].iCursor = -1; + sNC.pParse = pParse; + sNC.pSrcList = &sSrc; + sNC.isCheck = 1; + if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ + return; } } +#endif /* !defined(SQLITE_OMIT_CHECK) */ - /* Update the sqlite_sequence table by storing the content of the - ** maximum rowid counter values recorded while inserting into - ** autoincrement tables. + /* If the db->init.busy is 1 it means we are reading the SQL off the + ** "sqlite_master" or "sqlite_temp_master" table on the disk. + ** So do not write to the disk again. Extract the root page number + ** for the table from the db->init.newTnum field. (The page number + ** should have been put there by the sqliteOpenCb routine.) */ - if( pParse->nested==0 && pParse->pTriggerTab==0 ){ - sqlite3AutoincrementEnd(pParse); + if( db->init.busy ){ + p->tnum = db->init.newTnum; } - /* Return the number of rows that were deleted. If this routine is - ** generating code because of a call to sqlite3NestedParse(), do not - ** invoke the callback function. + /* If not initializing, then create a record for the new table + ** in the SQLITE_MASTER table of the database. + ** + ** If this is a TEMPORARY table, write the entry into the auxiliary + ** file instead of into the main database file. */ - if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); - } + if( !db->init.busy ){ + int n; + Vdbe *v; + char *zType; /* "view" or "table" */ + char *zType2; /* "VIEW" or "TABLE" */ + char *zStmt; /* Text of the CREATE TABLE or CREATE VIEW statement */ -delete_from_cleanup: - sqlite3AuthContextPop(&sContext); - sqlite3SrcListDelete(db, pTabList); - sqlite3ExprDelete(db, pWhere); - return; -} -/* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file -** (or in another file, if this file becomes part of the amalgamation). */ -#ifdef isView - #undef isView -#endif -#ifdef pTrigger - #undef pTrigger + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; + + sqlite3VdbeAddOp1(v, OP_Close, 0); + + /* + ** Initialize zType for the new view or table. + */ + if( p->pSelect==0 ){ + /* A regular table */ + zType = "table"; + zType2 = "TABLE"; +#ifndef SQLITE_OMIT_VIEW + }else{ + /* A view */ + zType = "view"; + zType2 = "VIEW"; #endif + } -/* -** This routine generates VDBE code that causes a single row of a -** single table to be deleted. -** -** The VDBE must be in a particular state when this routine is called. -** These are the requirements: -** -** 1. A read/write cursor pointing to pTab, the table containing the row -** to be deleted, must be opened as cursor number $iCur. -** -** 2. Read/write cursors for all indices of pTab must be open as -** cursor number base+i for the i-th index. -** -** 3. The record number of the row to be deleted must be stored in -** memory cell iRowid. -** -** This routine generates code to remove both the table record and all -** index entries that point to that record. -*/ -SQLITE_PRIVATE void sqlite3GenerateRowDelete( - Parse *pParse, /* Parsing context */ - Table *pTab, /* Table containing the row to be deleted */ - int iCur, /* Cursor number for the table */ - int iRowid, /* Memory cell that contains the rowid to delete */ - int count, /* If non-zero, increment the row change counter */ - Trigger *pTrigger, /* List of triggers to (potentially) fire */ - int onconf /* Default ON CONFLICT policy for triggers */ -){ - Vdbe *v = pParse->pVdbe; /* Vdbe */ - int iOld = 0; /* First register in OLD.* array */ - int iLabel; /* Label resolved to end of generated code */ + /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT + ** statement to populate the new table. The root-page number for the + ** new table is in register pParse->regRoot. + ** + ** Once the SELECT has been coded by sqlite3Select(), it is in a + ** suitable state to query for the column names and types to be used + ** by the new table. + ** + ** A shared-cache write-lock is not required to write to the new table, + ** as a schema-lock must have already been obtained to create it. Since + ** a schema-lock excludes all other database users, the write-lock would + ** be redundant. + */ + if( pSelect ){ + SelectDest dest; + Table *pSelTab; - /* Vdbe is guaranteed to have been allocated by this stage. */ - assert( v ); + assert(pParse->nTab==1); + sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeChangeP5(v, 1); + pParse->nTab = 2; + sqlite3SelectDestInit(&dest, SRT_Table, 1); + sqlite3Select(pParse, pSelect, &dest); + sqlite3VdbeAddOp1(v, OP_Close, 1); + if( pParse->nErr==0 ){ + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + } + } - /* Seek cursor iCur to the row to delete. If this row no longer exists - ** (this can happen if a trigger program has already deleted it), do - ** not attempt to delete it or fire any DELETE triggers. */ - iLabel = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, iLabel, iRowid); - - /* If there are any triggers to fire, allocate a range of registers to - ** use for the old.* references in the triggers. */ - if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ - u32 mask; /* Mask of OLD.* columns in use */ - int iCol; /* Iterator used while populating OLD.* */ + /* Compute the complete text of the CREATE statement */ + if( pSelect ){ + zStmt = createTableStmt(db, p); + }else{ + n = (int)(pEnd->z - pParse->sNameToken.z) + 1; + zStmt = sqlite3MPrintf(db, + "CREATE %s %.*s", zType2, n, pParse->sNameToken.z + ); + } - /* TODO: Could use temporary registers here. Also could attempt to - ** avoid copying the contents of the rowid register. */ - mask = sqlite3TriggerColmask( - pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf + /* A slot for the record has already been allocated in the + ** SQLITE_MASTER table. We just need to update that slot with all + ** the information we've collected. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s " + "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " + "WHERE rowid=#%d", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + zType, + p->zName, + p->zName, + pParse->regRoot, + zStmt, + pParse->regRowid ); - mask |= sqlite3FkOldmask(pParse, pTab); - iOld = pParse->nMem+1; - pParse->nMem += (1 + pTab->nCol); + sqlite3DbFree(db, zStmt); + sqlite3ChangeCookie(pParse, iDb); - /* Populate the OLD.* pseudo-table register array. These values will be - ** used by any BEFORE and AFTER triggers that exist. */ - sqlite3VdbeAddOp2(v, OP_Copy, iRowid, iOld); - for(iCol=0; iColnCol; iCol++){ - if( mask==0xffffffff || mask&(1<tabFlags & TF_Autoincrement ){ + Db *pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( pDb->pSchema->pSeqTab==0 ){ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.sqlite_sequence(name,seq)", + pDb->zName + ); } } +#endif - /* Invoke BEFORE DELETE trigger programs. */ - sqlite3CodeRowTrigger(pParse, pTrigger, - TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel - ); + /* Reparse everything to update our internal data structures */ + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); + } - /* Seek the cursor to the row to be deleted again. It may be that - ** the BEFORE triggers coded above have already removed the row - ** being deleted. Do not attempt to delete the row a second time, and - ** do not fire AFTER triggers. */ - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, iLabel, iRowid); - /* Do FK processing. This call checks that any FK constraints that - ** refer to this table (i.e. constraints attached to other tables) - ** are not violated by deleting this row. */ - sqlite3FkCheck(pParse, pTab, iOld, 0); - } + /* Add the table to the in-memory representation of the database. + */ + if( db->init.busy ){ + Table *pOld; + Schema *pSchema = p->pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, + sqlite3Strlen30(p->zName),p); + if( pOld ){ + assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ + db->mallocFailed = 1; + return; + } + pParse->pNewTable = 0; + db->nTable++; + db->flags |= SQLITE_InternChanges; - /* Delete the index and table entries. Skip this step if pTab is really - ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). */ - if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, 0); - sqlite3VdbeAddOp2(v, OP_Delete, iCur, (count?OPFLAG_NCHANGE:0)); - if( count ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC); +#ifndef SQLITE_OMIT_ALTERTABLE + if( !p->pSelect ){ + const char *zName = (const char *)pParse->sNameToken.z; + int nName; + assert( !pSelect && pCons && pEnd ); + if( pCons->z==0 ){ + pCons = pEnd; + } + nName = (int)((const char *)pCons->z - zName); + p->addColOffset = 13 + sqlite3Utf8CharLen(zName, nName); } +#endif } - - /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to - ** handle rows (possibly in other tables) that refer via a foreign key - ** to the row just deleted. */ - sqlite3FkActions(pParse, pTab, 0, iOld); - - /* Invoke AFTER DELETE trigger programs. */ - sqlite3CodeRowTrigger(pParse, pTrigger, - TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel - ); - - /* Jump here if the row had already been deleted before any BEFORE - ** trigger programs were invoked. Or if a trigger program throws a - ** RAISE(IGNORE) exception. */ - sqlite3VdbeResolveLabel(v, iLabel); } +#ifndef SQLITE_OMIT_VIEW /* -** This routine generates VDBE code that causes the deletion of all -** index entries associated with a single row of a single table. -** -** The VDBE must be in a particular state when this routine is called. -** These are the requirements: -** -** 1. A read/write cursor pointing to pTab, the table containing the row -** to be deleted, must be opened as cursor number "iCur". -** -** 2. Read/write cursors for all indices of pTab must be open as -** cursor number iCur+i for the i-th index. -** -** 3. The "iCur" cursor must be pointing to the row that is to be -** deleted. +** The parser calls this routine in order to create a new VIEW */ -SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( - Parse *pParse, /* Parsing and code generating context */ - Table *pTab, /* Table containing the row to be deleted */ - int iCur, /* Cursor number for the table */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ +SQLITE_PRIVATE void sqlite3CreateView( + Parse *pParse, /* The parsing context */ + Token *pBegin, /* The CREATE token that begins the statement */ + Token *pName1, /* The token that holds the name of the view */ + Token *pName2, /* The token that holds the name of the view */ + Select *pSelect, /* A SELECT statement that will become the new view */ + int isTemp, /* TRUE for a TEMPORARY view */ + int noErr /* Suppress error messages if VIEW already exists */ ){ - int i; - Index *pIdx; - int r1; + Table *p; + int n; + const char *z; + Token sEnd; + DbFixer sFix; + Token *pName; + int iDb; + sqlite3 *db = pParse->db; - for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - if( aRegIdx!=0 && aRegIdx[i-1]==0 ) continue; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iCur, 0, 0); - sqlite3VdbeAddOp3(pParse->pVdbe, OP_IdxDelete, iCur+i, r1,pIdx->nColumn+1); + if( pParse->nVar>0 ){ + sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); + sqlite3SelectDelete(db, pSelect); + return; } -} - -/* -** Generate code that will assemble an index key and put it in register -** regOut. The key with be for index pIdx which is an index on pTab. -** iCur is the index of a cursor open on the pTab table and pointing to -** the entry that needs indexing. -** -** Return a register number which is the first in a block of -** registers that holds the elements of the index key. The -** block of registers has already been deallocated by the time -** this routine returns. -*/ -SQLITE_PRIVATE int sqlite3GenerateIndexKey( - Parse *pParse, /* Parsing context */ - Index *pIdx, /* The index for which to generate a key */ - int iCur, /* Cursor number for the pIdx->pTable table */ - int regOut, /* Write the new index key to this register */ - int doMakeRec /* Run the OP_MakeRecord instruction if true */ -){ - Vdbe *v = pParse->pVdbe; - int j; - Table *pTab = pIdx->pTable; - int regBase; - int nCol; - - nCol = pIdx->nColumn; - regBase = sqlite3GetTempRange(pParse, nCol+1); - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regBase+nCol); - for(j=0; jaiColumn[j]; - if( idx==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regBase+nCol, regBase+j); - }else{ - sqlite3VdbeAddOp3(v, OP_Column, iCur, idx, regBase+j); - sqlite3ColumnDefault(v, pTab, idx, -1); - } + sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); + p = pParse->pNewTable; + if( p==0 || pParse->nErr ){ + sqlite3SelectDelete(db, pSelect); + return; } - if( doMakeRec ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0); + sqlite3TwoPartName(pParse, pName1, pName2, &pName); + iDb = sqlite3SchemaToIndex(db, p->pSchema); + if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) + && sqlite3FixSelect(&sFix, pSelect) + ){ + sqlite3SelectDelete(db, pSelect); + return; } - sqlite3ReleaseTempRange(pParse, regBase, nCol+1); - return regBase; -} -/************** End of delete.c **********************************************/ -/************** Begin file func.c ********************************************/ -/* -** 2002 February 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains the C functions that implement various SQL -** functions of SQLite. -** -** There is only one exported symbol in this file - the function -** sqliteRegisterBuildinFunctions() found at the bottom of the file. -** All other code has file scope. -*/ + /* Make a copy of the entire SELECT statement that defines the view. + ** This will force all the Expr.token.z values to be dynamically + ** allocated rather than point to the input string - which means that + ** they will persist after the current sqlite3_exec() call returns. + */ + p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed ){ + return; + } + if( !db->init.busy ){ + sqlite3ViewGetColumnNames(pParse, p); + } -/* -** Return the collating function associated with a function. -*/ -static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - return context->pColl; + /* Locate the end of the CREATE VIEW statement. Make sEnd point to + ** the end. + */ + sEnd = pParse->sLastToken; + if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + sEnd.z += sEnd.n; + } + sEnd.n = 0; + n = (int)(sEnd.z - pBegin->z); + z = pBegin->z; + while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + sEnd.z = &z[n-1]; + sEnd.n = 1; + + /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ + sqlite3EndTable(pParse, 0, &sEnd, 0); + return; } +#endif /* SQLITE_OMIT_VIEW */ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* -** Implementation of the non-aggregate min() and max() functions +** The Table structure pTable is really a VIEW. Fill in the names of +** the columns of the view in the pTable structure. Return the number +** of errors. If an error is seen leave an error message in pParse->zErrMsg. */ -static void minmaxFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - int mask; /* 0 for min() or 0xffffffff for max() */ - int iBest; - CollSeq *pColl; +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + Table *pSelTab; /* A fake table from which we get the result set */ + Select *pSel; /* Copy of the SELECT that implements the view */ + int nErr = 0; /* Number of errors encountered */ + int n; /* Temporarily holds the number of cursors assigned */ + sqlite3 *db = pParse->db; /* Database connection for malloc errors */ + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); - assert( argc>1 ); - mask = sqlite3_user_data(context)==0 ? 0 : -1; - pColl = sqlite3GetFuncCollSeq(context); - assert( pColl ); - assert( mask==-1 || mask==0 ); - iBest = 0; - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - for(i=1; i=0 ){ - testcase( mask==0 ); - iBest = i; - } - } - sqlite3_result_value(context, argv[iBest]); -} + assert( pTable ); -/* -** Return the type of the argument. -*/ -static void typeofFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - const char *z = 0; - UNUSED_PARAMETER(NotUsed); - switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_INTEGER: z = "integer"; break; - case SQLITE_TEXT: z = "text"; break; - case SQLITE_FLOAT: z = "real"; break; - case SQLITE_BLOB: z = "blob"; break; - default: z = "null"; break; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3VtabCallConnect(pParse, pTable) ){ + return SQLITE_ERROR; } - sqlite3_result_text(context, z, -1, SQLITE_STATIC); -} + if( IsVirtual(pTable) ) return 0; +#endif +#ifndef SQLITE_OMIT_VIEW + /* A positive nCol means the columns names for this view are + ** already known. + */ + if( pTable->nCol>0 ) return 0; -/* -** Implementation of the length() function -*/ -static void lengthFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int len; + /* A negative nCol is a special marker meaning that we are currently + ** trying to compute the column names. If we enter this routine with + ** a negative nCol, it means two or more views form a loop, like this: + ** + ** CREATE VIEW one AS SELECT * FROM two; + ** CREATE VIEW two AS SELECT * FROM one; + ** + ** Actually, the error above is now caught prior to reaching this point. + ** But the following test is still important as it does come up + ** in the following: + ** + ** CREATE TABLE main.ex1(a); + ** CREATE TEMP VIEW ex1 AS SELECT a FROM ex1; + ** SELECT * FROM temp.ex1; + */ + if( pTable->nCol<0 ){ + sqlite3ErrorMsg(pParse, "view %s is circularly defined", pTable->zName); + return 1; + } + assert( pTable->nCol>=0 ); - assert( argc==1 ); - UNUSED_PARAMETER(argc); - switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_BLOB: - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); - break; - } - case SQLITE_TEXT: { - const unsigned char *z = sqlite3_value_text(argv[0]); - if( z==0 ) return; - len = 0; - while( *z ){ - len++; - SQLITE_SKIP_UTF8(z); - } - sqlite3_result_int(context, len); - break; - } - default: { - sqlite3_result_null(context); - break; + /* If we get this far, it means we need to compute the table names. + ** Note that the call to sqlite3ResultSetOfSelect() will expand any + ** "*" elements in the results set of the view and will assign cursors + ** to the elements of the FROM clause. But we do not want these changes + ** to be permanent. So the computation is done on a copy of the SELECT + ** statement that defines the view. + */ + assert( pTable->pSelect ); + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + u8 enableLookaside = db->lookaside.bEnabled; + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + db->lookaside.bEnabled = 0; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; +#else + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); +#endif + db->lookaside.bEnabled = enableLookaside; + pParse->nTab = n; + if( pSelTab ){ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + pTable->pSchema->flags |= DB_UnresetViews; + }else{ + pTable->nCol = 0; + nErr++; } + sqlite3SelectDelete(db, pSel); + } else { + nErr++; } +#endif /* SQLITE_OMIT_VIEW */ + return nErr; } +#endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ +#ifndef SQLITE_OMIT_VIEW /* -** Implementation of the abs() function. -** -** IMP: R-23979-26855 The abs(X) function returns the absolute value of -** the numeric argument X. +** Clear the column names from every VIEW in database idx. */ -static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - assert( argc==1 ); - UNUSED_PARAMETER(argc); - switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_INTEGER: { - i64 iVal = sqlite3_value_int64(argv[0]); - if( iVal<0 ){ - if( (iVal<<1)==0 ){ - /* IMP: R-35460-15084 If X is the integer -9223372036854775807 then - ** abs(X) throws an integer overflow error since there is no - ** equivalent positive 64-bit two complement value. */ - sqlite3_result_error(context, "integer overflow", -1); - return; - } - iVal = -iVal; - } - sqlite3_result_int64(context, iVal); - break; - } - case SQLITE_NULL: { - /* IMP: R-37434-19929 Abs(X) returns NULL if X is NULL. */ - sqlite3_result_null(context); - break; - } - default: { - /* Because sqlite3_value_double() returns 0.0 if the argument is not - ** something that can be converted into a number, we have: - ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that - ** cannot be converted to a numeric value. - */ - double rVal = sqlite3_value_double(argv[0]); - if( rVal<0 ) rVal = -rVal; - sqlite3_result_double(context, rVal); - break; +static void sqliteViewResetAll(sqlite3 *db, int idx){ + HashElem *i; + assert( sqlite3SchemaMutexHeld(db, idx, 0) ); + if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; + for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + if( pTab->pSelect ){ + sqliteDeleteColumnNames(db, pTab); + pTab->aCol = 0; + pTab->nCol = 0; } } + DbClearProperty(db, idx, DB_UnresetViews); } +#else +# define sqliteViewResetAll(A,B) +#endif /* SQLITE_OMIT_VIEW */ /* -** Implementation of the substr() function. -** -** substr(x,p1,p2) returns p2 characters of x[] beginning with p1. -** p1 is 1-indexed. So substr(x,1,1) returns the first character -** of x. If x is text, then we actually count UTF-8 characters. -** If x is a blob, then we count bytes. -** -** If p1 is negative, then we begin abs(p1) from the end of x[]. +** This function is called by the VDBE to adjust the internal schema +** used by SQLite when the btree layer moves a table root page. The +** root-page of a table or index in database iDb has changed from iFrom +** to iTo. ** -** If p2 is negative, return the p2 characters preceeding p1. +** Ticket #1728: The symbol table might still contain information +** on tables and/or indices that are the process of being deleted. +** If you are unlucky, one of those deleted indices or tables might +** have the same rootpage number as the real table or index that is +** being moved. So we cannot stop searching after the first match +** because the first match might be for one of the deleted indices +** or tables and not the table/index that is actually being moved. +** We must continue looping until all tables and indices with +** rootpage==iFrom have been converted to have a rootpage of iTo +** in order to be certain that we got the right one. */ -static void substrFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *z; - const unsigned char *z2; - int len; - int p0type; - i64 p1, p2; - int negP2 = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iTo){ + HashElem *pElem; + Hash *pHash; + Db *pDb; - assert( argc==3 || argc==2 ); - if( sqlite3_value_type(argv[1])==SQLITE_NULL - || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) - ){ - return; - } - p0type = sqlite3_value_type(argv[0]); - p1 = sqlite3_value_int(argv[1]); - if( p0type==SQLITE_BLOB ){ - len = sqlite3_value_bytes(argv[0]); - z = sqlite3_value_blob(argv[0]); - if( z==0 ) return; - assert( len==sqlite3_value_bytes(argv[0]) ); - }else{ - z = sqlite3_value_text(argv[0]); - if( z==0 ) return; - len = 0; - if( p1<0 ){ - for(z2=z; *z2; len++){ - SQLITE_SKIP_UTF8(z2); - } - } - } - if( argc==3 ){ - p2 = sqlite3_value_int(argv[2]); - if( p2<0 ){ - p2 = -p2; - negP2 = 1; - } - }else{ - p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; - } - if( p1<0 ){ - p1 += len; - if( p1<0 ){ - p2 += p1; - if( p2<0 ) p2 = 0; - p1 = 0; - } - }else if( p1>0 ){ - p1--; - }else if( p2>0 ){ - p2--; - } - if( negP2 ){ - p1 -= p2; - if( p1<0 ){ - p2 += p1; - p1 = 0; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb = &db->aDb[iDb]; + pHash = &pDb->pSchema->tblHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + if( pTab->tnum==iFrom ){ + pTab->tnum = iTo; } } - assert( p1>=0 && p2>=0 ); - if( p0type!=SQLITE_BLOB ){ - while( *z && p1 ){ - SQLITE_SKIP_UTF8(z); - p1--; - } - for(z2=z; *z2 && p2; p2--){ - SQLITE_SKIP_UTF8(z2); - } - sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); - }else{ - if( p1+p2>len ){ - p2 = len-p1; - if( p2<0 ) p2 = 0; + pHash = &pDb->pSchema->idxHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Index *pIdx = sqliteHashData(pElem); + if( pIdx->tnum==iFrom ){ + pIdx->tnum = iTo; } - sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); } } +#endif /* -** Implementation of the round() function +** Write code to erase the table with root-page iTable from database iDb. +** Also write code to modify the sqlite_master table and internal schema +** if a root-page of another table is moved by the btree-layer whilst +** erasing iTable (this can happen with an auto-vacuum database). +*/ +static void destroyRootPage(Parse *pParse, int iTable, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); + sqlite3MayAbort(pParse); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* OP_Destroy stores an in integer r1. If this integer + ** is non-zero, then it is the root page number of a table moved to + ** location iTable. The following code modifies the sqlite_master table to + ** reflect this. + ** + ** The "#NNN" in the SQL is a special constant that means whatever value + ** is in register NNN. See grammar rules associated with the TK_REGISTER + ** token for additional information. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", + pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); +#endif + sqlite3ReleaseTempReg(pParse, r1); +} + +/* +** Write VDBE code to erase table pTab and all associated indices on disk. +** Code to update the sqlite_master tables and internal schema definitions +** in case a root-page belonging to another table is moved by the btree layer +** is also added (this can happen with an auto-vacuum database). */ -#ifndef SQLITE_OMIT_FLOATING_POINT -static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - int n = 0; - double r; - char *zBuf; - assert( argc==1 || argc==2 ); - if( argc==2 ){ - if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; - n = sqlite3_value_int(argv[1]); - if( n>30 ) n = 30; - if( n<0 ) n = 0; +static void destroyTable(Parse *pParse, Table *pTab){ +#ifdef SQLITE_OMIT_AUTOVACUUM + Index *pIdx; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + destroyRootPage(pParse, pTab->tnum, iDb); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + destroyRootPage(pParse, pIdx->tnum, iDb); } - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - r = sqlite3_value_double(argv[0]); - /* If Y==0 and X will fit in a 64-bit int, - ** handle the rounding directly, - ** otherwise use printf. +#else + /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM + ** is not defined), then it is important to call OP_Destroy on the + ** table and index root-pages in order, starting with the numerically + ** largest root-page number. This guarantees that none of the root-pages + ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the + ** following were coded: + ** + ** OP_Destroy 4 0 + ** ... + ** OP_Destroy 5 0 + ** + ** and root page 5 happened to be the largest root-page number in the + ** database, then root page 5 would be moved to page 4 by the + ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit + ** a free-list page. */ - if( n==0 && r>=0 && rtnum; + int iDestroyed = 0; + + while( 1 ){ + Index *pIdx; + int iLargest = 0; + + if( iDestroyed==0 || iTabpIndex; pIdx; pIdx=pIdx->pNext){ + int iIdx = pIdx->tnum; + assert( pIdx->pSchema==pTab->pSchema ); + if( (iDestroyed==0 || (iIdxiLargest ){ + iLargest = iIdx; + } + } + if( iLargest==0 ){ return; + }else{ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + destroyRootPage(pParse, iLargest, iDb); + iDestroyed = iLargest; } - sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); - sqlite3_free(zBuf); } - sqlite3_result_double(context, r); -} #endif +} /* -** Allocate nByte bytes of space using sqlite3_malloc(). If the -** allocation fails, call sqlite3_result_error_nomem() to notify -** the database handle that malloc() has failed and return NULL. -** If nByte is larger than the maximum string or blob length, then -** raise an SQLITE_TOOBIG exception and return NULL. +** This routine is called to do the work of a DROP TABLE statement. +** pName is the name of the table to be dropped. */ -static void *contextMalloc(sqlite3_context *context, i64 nByte){ - char *z; - sqlite3 *db = sqlite3_context_db_handle(context); - assert( nByte>0 ); - testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); - if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - z = 0; - }else{ - z = sqlite3Malloc((int)nByte); - if( !z ){ - sqlite3_result_error_nomem(context); - } +SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ + Table *pTab; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + if( db->mallocFailed ){ + goto exit_drop_table; } - return z; -} + assert( pParse->nErr==0 ); + assert( pName->nSrc==1 ); + if( noErr ) db->suppressErr++; + pTab = sqlite3LocateTable(pParse, isView, + pName->a[0].zName, pName->a[0].zDatabase); + if( noErr ) db->suppressErr--; -/* -** Implementation of the upper() and lower() SQL functions. -*/ -static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - char *z1; - const char *z2; - int i, n; - UNUSED_PARAMETER(argc); - z2 = (char*)sqlite3_value_text(argv[0]); - n = sqlite3_value_bytes(argv[0]); - /* Verify that the call to _bytes() does not invalidate the _text() pointer */ - assert( z2==(char*)sqlite3_value_text(argv[0]) ); - if( z2 ){ - z1 = contextMalloc(context, ((i64)n)+1); - if( z1 ){ - memcpy(z1, z2, n+1); - for(i=0; z1[i]; i++){ - z1[i] = (char)sqlite3Toupper(z1[i]); - } - sqlite3_result_text(context, z1, -1, sqlite3_free); - } + if( pTab==0 ){ + if( noErr ) sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + goto exit_drop_table; } -} -static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - u8 *z1; - const char *z2; - int i, n; - UNUSED_PARAMETER(argc); - z2 = (char*)sqlite3_value_text(argv[0]); - n = sqlite3_value_bytes(argv[0]); - /* Verify that the call to _bytes() does not invalidate the _text() pointer */ - assert( z2==(char*)sqlite3_value_text(argv[0]) ); - if( z2 ){ - z1 = contextMalloc(context, ((i64)n)+1); - if( z1 ){ - memcpy(z1, z2, n+1); - for(i=0; z1[i]; i++){ - z1[i] = sqlite3Tolower(z1[i]); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbnDb ); + + /* If pTab is a virtual table, call ViewGetColumnNames() to ensure + ** it is initialized. + */ + if( IsVirtual(pTab) && sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_drop_table; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code; + const char *zTab = SCHEMA_TABLE(iDb); + const char *zDb = db->aDb[iDb].zName; + const char *zArg2 = 0; + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ + goto exit_drop_table; + } + if( isView ){ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_VIEW; + }else{ + code = SQLITE_DROP_VIEW; } - sqlite3_result_text(context, (char *)z1, -1, sqlite3_free); +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( IsVirtual(pTab) ){ + code = SQLITE_DROP_VTABLE; + zArg2 = sqlite3GetVTable(db, pTab)->pMod->zName; +#endif + }else{ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_TABLE; + }else{ + code = SQLITE_DROP_TABLE; + } + } + if( sqlite3AuthCheck(pParse, code, pTab->zName, zArg2, zDb) ){ + goto exit_drop_table; + } + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){ + goto exit_drop_table; } } -} +#endif + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); + goto exit_drop_table; + } +#ifndef SQLITE_OMIT_VIEW + /* Ensure DROP TABLE is not used on a view, and DROP VIEW is not used + ** on a table. + */ + if( isView && pTab->pSelect==0 ){ + sqlite3ErrorMsg(pParse, "use DROP TABLE to delete table %s", pTab->zName); + goto exit_drop_table; + } + if( !isView && pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "use DROP VIEW to delete view %s", pTab->zName); + goto exit_drop_table; + } +#endif -#if 0 /* This function is never used. */ -/* -** The COALESCE() and IFNULL() functions used to be implemented as shown -** here. But now they are implemented as VDBE code so that unused arguments -** do not have to be computed. This legacy implementation is retained as -** comment. -*/ -/* -** Implementation of the IFNULL(), NVL(), and COALESCE() functions. -** All three do the same thing. They return the first non-NULL -** argument. -*/ -static void ifnullFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - for(i=0; iaDb[iDb]; + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp0(v, OP_VBegin); } - } -} -#endif /* NOT USED */ -#define ifnullFunc versionFunc /* Substitute function - never called */ +#endif + sqlite3FkDropTable(pParse, pName, pTab); -/* -** Implementation of random(). Return a random integer. -*/ -static void randomFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **NotUsed2 -){ - sqlite_int64 r; - UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_randomness(sizeof(r), &r); - if( r<0 ){ - /* We need to prevent a random number of 0x8000000000000000 - ** (or -9223372036854775808) since when you do abs() of that - ** number of you get the same value back again. To do this - ** in a way that is testable, mask the sign bit off of negative - ** values, resulting in a positive value. Then take the - ** 2s complement of that positive value. The end result can - ** therefore be no less than -9223372036854775807. + /* Drop all triggers associated with the table being dropped. Code + ** is generated to remove entries from sqlite_master and/or + ** sqlite_temp_master if required. */ - r = -(r ^ (((sqlite3_int64)1)<<63)); + pTrigger = sqlite3TriggerList(pParse, pTab); + while( pTrigger ){ + assert( pTrigger->pSchema==pTab->pSchema || + pTrigger->pSchema==db->aDb[1].pSchema ); + sqlite3DropTriggerPtr(pParse, pTrigger); + pTrigger = pTrigger->pNext; + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Remove any entries of the sqlite_sequence table associated with + ** the table being dropped. This is done before the table is dropped + ** at the btree level, in case the sqlite_sequence table needs to + ** move as a result of the drop (can happen in auto-vacuum mode). + */ + if( pTab->tabFlags & TF_Autoincrement ){ + sqlite3NestedParse(pParse, + "DELETE FROM %s.sqlite_sequence WHERE name=%Q", + pDb->zName, pTab->zName + ); + } +#endif + + /* Drop all SQLITE_MASTER table and index entries that refer to the + ** table. The program name loops through the master table and deletes + ** every row that refers to a table of the same name as the one being + ** dropped. Triggers are handled seperately because a trigger can be + ** created in the temp database that refers to a table in another + ** database. + */ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", + pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + + /* Drop any statistics from the sqlite_stat1 table, if it exists */ + if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q", pDb->zName, pTab->zName + ); + } + + if( !isView && !IsVirtual(pTab) ){ + destroyTable(pParse, pTab); + } + + /* Remove the table entry from SQLite's internal schema and modify + ** the schema cookie. + */ + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + } + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); } - sqlite3_result_int64(context, r); + sqliteViewResetAll(db, iDb); + +exit_drop_table: + sqlite3SrcListDelete(db, pName); } /* -** Implementation of randomblob(N). Return a random blob -** that is N bytes long. +** This routine is called to create a new foreign key on the table +** currently under construction. pFromCol determines which columns +** in the current table point to the foreign key. If pFromCol==0 then +** connect the key to the last column inserted. pTo is the name of +** the table referred to. pToCol is a list of tables in the other +** pTo table that the foreign key points to. flags contains all +** information about the conflict resolution algorithms specified +** in the ON DELETE, ON UPDATE and ON INSERT clauses. +** +** An FKey structure is created and added to the table currently +** under construction in the pParse->pNewTable field. +** +** The foreign key is set for IMMEDIATE processing. A subsequent call +** to sqlite3DeferForeignKey() might change this to DEFERRED. */ -static void randomBlob( - sqlite3_context *context, - int argc, - sqlite3_value **argv +SQLITE_PRIVATE void sqlite3CreateForeignKey( + Parse *pParse, /* Parsing context */ + ExprList *pFromCol, /* Columns in this table that point to other table */ + Token *pTo, /* Name of the other table */ + ExprList *pToCol, /* Columns in the other table */ + int flags /* Conflict resolution algorithms. */ ){ - int n; - unsigned char *p; - assert( argc==1 ); - UNUSED_PARAMETER(argc); - n = sqlite3_value_int(argv[0]); - if( n<1 ){ - n = 1; + sqlite3 *db = pParse->db; +#ifndef SQLITE_OMIT_FOREIGN_KEY + FKey *pFKey = 0; + FKey *pNextTo; + Table *p = pParse->pNewTable; + int nByte; + int i; + int nCol; + char *z; + + assert( pTo!=0 ); + if( p==0 || IN_DECLARE_VTAB ) goto fk_end; + if( pFromCol==0 ){ + int iCol = p->nCol-1; + if( NEVER(iCol<0) ) goto fk_end; + if( pToCol && pToCol->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "foreign key on %s" + " should reference only one column of table %T", + p->aCol[iCol].zName, pTo); + goto fk_end; + } + nCol = 1; + }else if( pToCol && pToCol->nExpr!=pFromCol->nExpr ){ + sqlite3ErrorMsg(pParse, + "number of columns in foreign key does not match the number of " + "columns in the referenced table"); + goto fk_end; + }else{ + nCol = pFromCol->nExpr; } - p = contextMalloc(context, n); - if( p ){ - sqlite3_randomness(n, p); - sqlite3_result_blob(context, (char*)p, n, sqlite3_free); + nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + if( pToCol ){ + for(i=0; inExpr; i++){ + nByte += sqlite3Strlen30(pToCol->a[i].zName) + 1; + } } -} + pFKey = sqlite3DbMallocZero(db, nByte ); + if( pFKey==0 ){ + goto fk_end; + } + pFKey->pFrom = p; + pFKey->pNextFrom = p->pFKey; + z = (char*)&pFKey->aCol[nCol]; + pFKey->zTo = z; + memcpy(z, pTo->z, pTo->n); + z[pTo->n] = 0; + sqlite3Dequote(z); + z += pTo->n+1; + pFKey->nCol = nCol; + if( pFromCol==0 ){ + pFKey->aCol[0].iFrom = p->nCol-1; + }else{ + for(i=0; inCol; j++){ + if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ + pFKey->aCol[i].iFrom = j; + break; + } + } + if( j>=p->nCol ){ + sqlite3ErrorMsg(pParse, + "unknown column \"%s\" in foreign key definition", + pFromCol->a[i].zName); + goto fk_end; + } + } + } + if( pToCol ){ + for(i=0; ia[i].zName); + pFKey->aCol[i].zCol = z; + memcpy(z, pToCol->a[i].zName, n); + z[n] = 0; + z += n+1; + } + } + pFKey->isDeferred = 0; + pFKey->aAction[0] = (u8)(flags & 0xff); /* ON DELETE action */ + pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ -/* -** Implementation of the last_insert_rowid() SQL function. The return -** value is the same as the sqlite3_last_insert_rowid() API function. -*/ -static void last_insert_rowid( - sqlite3_context *context, - int NotUsed, - sqlite3_value **NotUsed2 -){ - sqlite3 *db = sqlite3_context_db_handle(context); - UNUSED_PARAMETER2(NotUsed, NotUsed2); - /* IMP: R-51513-12026 The last_insert_rowid() SQL function is a - ** wrapper around the sqlite3_last_insert_rowid() C/C++ interface - ** function. */ - sqlite3_result_int64(context, sqlite3_last_insert_rowid(db)); -} + assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, + pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey + ); + if( pNextTo==pFKey ){ + db->mallocFailed = 1; + goto fk_end; + } + if( pNextTo ){ + assert( pNextTo->pPrevTo==0 ); + pFKey->pNextTo = pNextTo; + pNextTo->pPrevTo = pFKey; + } -/* -** Implementation of the changes() SQL function. -** -** IMP: R-62073-11209 The changes() SQL function is a wrapper -** around the sqlite3_changes() C/C++ function and hence follows the same -** rules for counting changes. -*/ -static void changes( - sqlite3_context *context, - int NotUsed, - sqlite3_value **NotUsed2 -){ - sqlite3 *db = sqlite3_context_db_handle(context); - UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_result_int(context, sqlite3_changes(db)); + /* Link the foreign key to the table as the last step. + */ + p->pFKey = pFKey; + pFKey = 0; + +fk_end: + sqlite3DbFree(db, pFKey); +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + sqlite3ExprListDelete(db, pFromCol); + sqlite3ExprListDelete(db, pToCol); } /* -** Implementation of the total_changes() SQL function. The return value is -** the same as the sqlite3_total_changes() API function. +** This routine is called when an INITIALLY IMMEDIATE or INITIALLY DEFERRED +** clause is seen as part of a foreign key definition. The isDeferred +** parameter is 1 for INITIALLY DEFERRED and 0 for INITIALLY IMMEDIATE. +** The behavior of the most recently created foreign key is adjusted +** accordingly. */ -static void total_changes( - sqlite3_context *context, - int NotUsed, - sqlite3_value **NotUsed2 -){ - sqlite3 *db = sqlite3_context_db_handle(context); - UNUSED_PARAMETER2(NotUsed, NotUsed2); - /* IMP: R-52756-41993 This function is a wrapper around the - ** sqlite3_total_changes() C/C++ interface. */ - sqlite3_result_int(context, sqlite3_total_changes(db)); +SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ +#ifndef SQLITE_OMIT_FOREIGN_KEY + Table *pTab; + FKey *pFKey; + if( (pTab = pParse->pNewTable)==0 || (pFKey = pTab->pFKey)==0 ) return; + assert( isDeferred==0 || isDeferred==1 ); /* EV: R-30323-21917 */ + pFKey->isDeferred = (u8)isDeferred; +#endif } /* -** A structure defining how to do GLOB-style comparisons. +** Generate code that will erase and refill index *pIdx. This is +** used to initialize a newly created index or to recompute the +** content of an index in response to a REINDEX command. +** +** if memRootPage is not negative, it means that the index is newly +** created. The register specified by memRootPage contains the +** root page number of the index. If memRootPage is negative, then +** the index already exists and must be cleared before being refilled and +** the root page number of the index is taken from pIndex->tnum. */ -struct compareInfo { - u8 matchAll; - u8 matchOne; - u8 matchSet; - u8 noCase; -}; +static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ + Table *pTab = pIndex->pTable; /* The table that is indexed */ + int iTab = pParse->nTab++; /* Btree cursor used for pTab */ + int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ + int addr1; /* Address of top of loop */ + int tnum; /* Root page of index */ + Vdbe *v; /* Generate code into this virtual machine */ + KeyInfo *pKey; /* KeyInfo for index */ + int regIdxKey; /* Registers containing the index key */ + int regRecord; /* Register holding assemblied index record */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); -/* -** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, all characters are -** able to participate in upper-case-to-lower-case mappings in EBCDIC -** whereas only characters less than 0x80 do in ASCII. -*/ -#if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A,C) (*(A++)) -# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] -#else -# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, + db->aDb[iDb].zName ) ){ + return; + } #endif -static const struct compareInfo globInfo = { '*', '?', '[', 0 }; -/* The correct SQL-92 behavior is for the LIKE operator to ignore -** case. Thus 'a' LIKE 'A' would be true. */ -static const struct compareInfo likeInfoNorm = { '%', '_', 0, 1 }; -/* If SQLITE_CASE_SENSITIVE_LIKE is defined, then the LIKE operator -** is case sensitive causing 'a' LIKE 'A' to be false */ -static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; + /* Require a write-lock on the table to perform this operation */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); -/* -** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" expression. Return true (1) if they -** are the same and false (0) if they are different. -** -** Globbing rules: -** -** '*' Matches any sequence of zero or more characters. -** -** '?' Matches exactly one character. -** -** [...] Matches one character from the enclosed list of -** characters. -** -** [^...] Matches one character not in the enclosed list. -** -** With the [...] and [^...] matching, a ']' character can be included -** in the list by making it the first character after '[' or '^'. A -** range of characters can be specified using '-'. Example: -** "[a-z]" matches any single lower-case letter. To match a '-', make -** it the last character in the list. -** -** This routine is usually quick, but can be N**2 in the worst case. -** -** Hints: to match '*' or '?', put them in "[]". Like this: -** -** abc[*]xyz Matches "abc*xyz" only -*/ -static int patternCompare( - const u8 *zPattern, /* The glob pattern */ - const u8 *zString, /* The string to compare against the glob */ - const struct compareInfo *pInfo, /* Information about how to do the compare */ - const int esc /* The escape character */ -){ - int c, c2; - int invert; - int seen; - u8 matchOne = pInfo->matchOne; - u8 matchAll = pInfo->matchAll; - u8 matchSet = pInfo->matchSet; - u8 noCase = pInfo->noCase; - int prevEscape = 0; /* True if the previous character was 'escape' */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) return; + if( memRootPage>=0 ){ + tnum = memRootPage; + }else{ + tnum = pIndex->tnum; + sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); + } + pKey = sqlite3IndexKeyinfo(pParse, pIndex); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, + (char *)pKey, P4_KEYINFO_HANDOFF); + if( memRootPage>=0 ){ + sqlite3VdbeChangeP5(v, 1); + } + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + regRecord = sqlite3GetTempReg(pParse); + regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); + if( pIndex->onError!=OE_None ){ + const int regRowid = regIdxKey + pIndex->nColumn; + const int j2 = sqlite3VdbeCurrentAddr(v) + 2; + void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); - while( (c = sqlite3Utf8Read(zPattern,&zPattern))!=0 ){ - if( !prevEscape && c==matchAll ){ - while( (c=sqlite3Utf8Read(zPattern,&zPattern)) == matchAll - || c == matchOne ){ - if( c==matchOne && sqlite3Utf8Read(zString, &zString)==0 ){ - return 0; - } - } - if( c==0 ){ - return 1; - }else if( c==esc ){ - c = sqlite3Utf8Read(zPattern, &zPattern); - if( c==0 ){ - return 0; - } - }else if( c==matchSet ){ - assert( esc==0 ); /* This is GLOB, not LIKE */ - assert( matchSet<0x80 ); /* '[' is a single-byte character */ - while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ - SQLITE_SKIP_UTF8(zString); - } - return *zString!=0; - } - while( (c2 = sqlite3Utf8Read(zString,&zString))!=0 ){ - if( noCase ){ - GlogUpperToLower(c2); - GlogUpperToLower(c); - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); - GlogUpperToLower(c2); - } - }else{ - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(zString, &zString); - } - } - if( c2==0 ) return 0; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; - } - return 0; - }else if( !prevEscape && c==matchOne ){ - if( sqlite3Utf8Read(zString, &zString)==0 ){ - return 0; - } - }else if( c==matchSet ){ - int prior_c = 0; - assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ - seen = 0; - invert = 0; - c = sqlite3Utf8Read(zString, &zString); - if( c==0 ) return 0; - c2 = sqlite3Utf8Read(zPattern, &zPattern); - if( c2=='^' ){ - invert = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); - } - if( c2==']' ){ - if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(zPattern, &zPattern); - } - while( c2 && c2!=']' ){ - if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(zPattern, &zPattern); - if( c>=prior_c && c<=c2 ) seen = 1; - prior_c = 0; - }else{ - if( c==c2 ){ - seen = 1; - } - prior_c = c2; - } - c2 = sqlite3Utf8Read(zPattern, &zPattern); - } - if( c2==0 || (seen ^ invert)==0 ){ - return 0; - } - }else if( esc==c && !prevEscape ){ - prevEscape = 1; - }else{ - c2 = sqlite3Utf8Read(zString, &zString); - if( noCase ){ - GlogUpperToLower(c); - GlogUpperToLower(c2); - } - if( c!=c2 ){ - return 0; - } - prevEscape = 0; - } + /* The registers accessed by the OP_IsUnique opcode were allocated + ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() + ** call above. Just before that function was freed they were released + ** (made available to the compiler for reuse) using + ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique + ** opcode use the values stored within seems dangerous. However, since + ** we can be sure that no other temp registers have been allocated + ** since sqlite3ReleaseTempRange() was called, it is safe to do so. + */ + sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); } - return *zString==0; + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_Close, iTab); + sqlite3VdbeAddOp1(v, OP_Close, iIdx); } /* -** Count the number of times that the LIKE operator (or GLOB which is -** just a variation of LIKE) gets called. This is used for testing -** only. -*/ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_like_count = 0; -#endif - - -/* -** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the -** pattern and the second argument is the string. So, the SQL statements: -** -** A LIKE B +** Create a new index for an SQL table. pName1.pName2 is the name of the index +** and pTblList is the name of the table that is to be indexed. Both will +** be NULL for a primary key or an index that is created to satisfy a +** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable +** as the table to be indexed. pParse->pNewTable is a table that is +** currently being constructed by a CREATE TABLE statement. ** -** is implemented as like(B,A). +** pList is a list of columns to be indexed. pList will be NULL if this +** is a primary key or unique-constraint on the most recent column added +** to the table currently under construction. ** -** This same function (with a different compareInfo structure) computes -** the GLOB operator. +** If the index is created successfully, return a pointer to the new Index +** structure. This is used by sqlite3AddPrimaryKey() to mark the index +** as the tables primary key (Index.autoIndex==2). */ -static void likeFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv +SQLITE_PRIVATE Index *sqlite3CreateIndex( + Parse *pParse, /* All information about this parse */ + Token *pName1, /* First part of index name. May be NULL */ + Token *pName2, /* Second part of index name. May be NULL */ + SrcList *pTblName, /* Table to index. Use pParse->pNewTable if 0 */ + ExprList *pList, /* A list of columns to be indexed */ + int onError, /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + Token *pStart, /* The CREATE token that begins this statement */ + Token *pEnd, /* The ")" that closes the CREATE INDEX statement */ + int sortOrder, /* Sort order of primary key when pList==NULL */ + int ifNotExist /* Omit error if index already exists */ ){ - const unsigned char *zA, *zB; - int escape = 0; - int nPat; - sqlite3 *db = sqlite3_context_db_handle(context); + Index *pRet = 0; /* Pointer to return */ + Table *pTab = 0; /* Table to be indexed */ + Index *pIndex = 0; /* The index to be created */ + char *zName = 0; /* Name of the index */ + int nName; /* Number of characters in zName */ + int i, j; + Token nullId; /* Fake token for an empty ID list */ + DbFixer sFix; /* For assigning database names to pTable */ + int sortOrderMask; /* 1 to honor DESC in index. 0 to ignore. */ + sqlite3 *db = pParse->db; + Db *pDb; /* The specific table containing the indexed database */ + int iDb; /* Index of the database that is being written */ + Token *pName = 0; /* Unqualified name of the index to create */ + struct ExprList_item *pListItem; /* For looping over pList */ + int nCol; + int nExtra = 0; + char *zExtra; - zB = sqlite3_value_text(argv[0]); - zA = sqlite3_value_text(argv[1]); + assert( pStart==0 || pEnd!=0 ); /* pEnd must be non-NULL if pStart is */ + assert( pParse->nErr==0 ); /* Never called with prior errors */ + if( db->mallocFailed || IN_DECLARE_VTAB ){ + goto exit_create_index; + } + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_create_index; + } - /* Limit the length of the LIKE or GLOB pattern to avoid problems - ** of deep recursion and N*N behavior in patternCompare(). + /* + ** Find the table that is to be indexed. Return early if not found. */ - nPat = sqlite3_value_bytes(argv[0]); - testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ); - testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]+1 ); - if( nPat > db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ){ - sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); - return; - } - assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ + if( pTblName!=0 ){ - if( argc==3 ){ - /* The escape character string must consist of a single UTF-8 character. - ** Otherwise, return an error. + /* Use the two-part index name to determine the database + ** to search for the table. 'Fix' the table name to this db + ** before looking up the table. */ - const unsigned char *zEsc = sqlite3_value_text(argv[2]); - if( zEsc==0 ) return; - if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ - sqlite3_result_error(context, - "ESCAPE expression must be a single character", -1); - return; + assert( pName1 && pName2 ); + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) goto exit_create_index; + +#ifndef SQLITE_OMIT_TEMPDB + /* If the index name was unqualified, check if the the table + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. + */ + if( !db->init.busy ){ + pTab = sqlite3SrcListLookup(pParse, pTblName); + if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } } - escape = sqlite3Utf8Read(zEsc, &zEsc); - } - if( zA && zB ){ - struct compareInfo *pInfo = sqlite3_user_data(context); -#ifdef SQLITE_TEST - sqlite3_like_count++; #endif - - sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); - } -} -/* -** Implementation of the NULLIF(x,y) function. The result is the first -** argument if the arguments are different. The result is NULL if the -** arguments are equal to each other. -*/ -static void nullifFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - CollSeq *pColl = sqlite3GetFuncCollSeq(context); - UNUSED_PARAMETER(NotUsed); - if( sqlite3MemCompare(argv[0], argv[1], pColl)!=0 ){ - sqlite3_result_value(context, argv[0]); + if( sqlite3FixInit(&sFix, pParse, iDb, "index", pName) && + sqlite3FixSrcList(&sFix, pTblName) + ){ + /* Because the parser constructs pTblName from a single identifier, + ** sqlite3FixSrcList can never fail. */ + assert(0); + } + pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, + pTblName->a[0].zDatabase); + if( !pTab || db->mallocFailed ) goto exit_create_index; + assert( db->aDb[iDb].pSchema==pTab->pSchema ); + }else{ + assert( pName==0 ); + pTab = pParse->pNewTable; + if( !pTab ) goto exit_create_index; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); } -} - -/* -** Implementation of the sqlite_version() function. The result is the version -** of the SQLite library that is running. -*/ -static void versionFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **NotUsed2 -){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - /* IMP: R-48699-48617 This function is an SQL wrapper around the - ** sqlite3_libversion() C-interface. */ - sqlite3_result_text(context, sqlite3_libversion(), -1, SQLITE_STATIC); -} - -/* -** Implementation of the sqlite_source_id() function. The result is a string -** that identifies the particular version of the source code used to build -** SQLite. -*/ -static void sourceidFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **NotUsed2 -){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - /* IMP: R-24470-31136 This function is an SQL wrapper around the - ** sqlite3_sourceid() C interface. */ - sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); -} + pDb = &db->aDb[iDb]; -/* -** Implementation of the sqlite_compileoption_used() function. -** The result is an integer that identifies if the compiler option -** was used to build SQLite. -*/ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS -static void compileoptionusedFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zOptName; - assert( argc==1 ); - UNUSED_PARAMETER(argc); - /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL - ** function is a wrapper around the sqlite3_compileoption_used() C/C++ - ** function. - */ - if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ - sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); + assert( pTab!=0 ); + assert( pParse->nErr==0 ); + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && memcmp(&pTab->zName[7],"altertab_",9)!=0 ){ + sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); + goto exit_create_index; } -} -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "views may not be indexed"); + goto exit_create_index; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be indexed"); + goto exit_create_index; + } +#endif -/* -** Implementation of the sqlite_compileoption_get() function. -** The result is a string that identifies the compiler options -** used to build SQLite. -*/ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS -static void compileoptiongetFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int n; - assert( argc==1 ); - UNUSED_PARAMETER(argc); - /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function - ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + /* + ** Find the name of the index. Make sure there is not already another + ** index or table with the same name. + ** + ** Exception: If we are reading the names of permanent indices from the + ** sqlite_master table (because some other process changed the schema) and + ** one of the index names collides with the name of a temporary table or + ** index, then we will continue to process this index. + ** + ** If pName==0 it means that we are + ** dealing with a primary key or UNIQUE constraint. We have to invent our + ** own name. */ - n = sqlite3_value_int(argv[0]); - sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); -} -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ - -/* Array for converting from half-bytes (nybbles) into ASCII hex -** digits. */ -static const char hexdigits[] = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' -}; - -/* -** EXPERIMENTAL - This is not an official function. The interface may -** change. This function may disappear. Do not write code that depends -** on this function. -** -** Implementation of the QUOTE() function. This function takes a single -** argument. If the argument is numeric, the return value is the same as -** the argument. If the argument is NULL, the return value is the string -** "NULL". Otherwise, the argument is enclosed in single quotes with -** single-quote escapes. -*/ -static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - assert( argc==1 ); - UNUSED_PARAMETER(argc); - switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - sqlite3_result_value(context, argv[0]); - break; + if( pName ){ + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ) goto exit_create_index; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + goto exit_create_index; } - case SQLITE_BLOB: { - char *zText = 0; - char const *zBlob = sqlite3_value_blob(argv[0]); - int nBlob = sqlite3_value_bytes(argv[0]); - assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ - zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); - if( zText ){ - int i; - for(i=0; i>4)&0x0F]; - zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; - } - zText[(nBlob*2)+2] = '\''; - zText[(nBlob*2)+3] = '\0'; - zText[0] = 'X'; - zText[1] = '\''; - sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); - sqlite3_free(zText); + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, 0)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; } - break; } - case SQLITE_TEXT: { - int i,j; - u64 n; - const unsigned char *zArg = sqlite3_value_text(argv[0]); - char *z; - - if( zArg==0 ) return; - for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } - z = contextMalloc(context, ((i64)i)+((i64)n)+3); - if( z ){ - z[0] = '\''; - for(i=0, j=1; zArg[i]; i++){ - z[j++] = zArg[i]; - if( zArg[i]=='\'' ){ - z[j++] = '\''; - } - } - z[j++] = '\''; - z[j] = 0; - sqlite3_result_text(context, z, j, sqlite3_free); + if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); } - break; + goto exit_create_index; } - default: { - assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); - sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); - break; + }else{ + int n; + Index *pLoop; + for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} + zName = sqlite3MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); + if( zName==0 ){ + goto exit_create_index; } } -} -/* -** The hex() function. Interpret the argument as a blob. Return -** a hexadecimal rendering as text. -*/ -static void hexFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i, n; - const unsigned char *pBlob; - char *zHex, *z; - assert( argc==1 ); - UNUSED_PARAMETER(argc); - pBlob = sqlite3_value_blob(argv[0]); - n = sqlite3_value_bytes(argv[0]); - assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ - z = zHex = contextMalloc(context, ((i64)n)*2 + 1); - if( zHex ){ - for(i=0; i>4)&0xf]; - *(z++) = hexdigits[c&0xf]; + /* Check for authorization to create an index. + */ +#ifndef SQLITE_OMIT_AUTHORIZATION + { + const char *zDb = pDb->zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ + goto exit_create_index; + } + i = SQLITE_CREATE_INDEX; + if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE_CREATE_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ + goto exit_create_index; } - *z = 0; - sqlite3_result_text(context, zHex, n*2, sqlite3_free); } -} +#endif -/* -** The zeroblob(N) function returns a zero-filled blob of size N bytes. -*/ -static void zeroblobFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - i64 n; - sqlite3 *db = sqlite3_context_db_handle(context); - assert( argc==1 ); - UNUSED_PARAMETER(argc); - n = sqlite3_value_int64(argv[0]); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); - if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - }else{ - sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + /* If pList==0, it means this routine was called to make a primary + ** key out of the last column added to the table under construction. + ** So create a fake list to simulate this. + */ + if( pList==0 ){ + nullId.z = pTab->aCol[pTab->nCol-1].zName; + nullId.n = sqlite3Strlen30((char*)nullId.z); + pList = sqlite3ExprListAppend(pParse, 0, 0); + if( pList==0 ) goto exit_create_index; + sqlite3ExprListSetName(pParse, pList, &nullId, 0); + pList->a[0].sortOrder = (u8)sortOrder; } -} - -/* -** The replace() function. Three arguments are all strings: call -** them A, B, and C. The result is also a string which is derived -** from A by replacing every occurance of B with C. The match -** must be exact. Collating sequences are not used. -*/ -static void replaceFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zStr; /* The input string A */ - const unsigned char *zPattern; /* The pattern string B */ - const unsigned char *zRep; /* The replacement string C */ - unsigned char *zOut; /* The output */ - int nStr; /* Size of zStr */ - int nPattern; /* Size of zPattern */ - int nRep; /* Size of zRep */ - i64 nOut; /* Maximum size of zOut */ - int loopLimit; /* Last zStr[] that might match zPattern[] */ - int i, j; /* Loop counters */ - assert( argc==3 ); - UNUSED_PARAMETER(argc); - zStr = sqlite3_value_text(argv[0]); - if( zStr==0 ) return; - nStr = sqlite3_value_bytes(argv[0]); - assert( zStr==sqlite3_value_text(argv[0]) ); /* No encoding change */ - zPattern = sqlite3_value_text(argv[1]); - if( zPattern==0 ){ - assert( sqlite3_value_type(argv[1])==SQLITE_NULL - || sqlite3_context_db_handle(context)->mallocFailed ); - return; - } - if( zPattern[0]==0 ){ - assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); - sqlite3_result_value(context, argv[0]); - return; - } - nPattern = sqlite3_value_bytes(argv[1]); - assert( zPattern==sqlite3_value_text(argv[1]) ); /* No encoding change */ - zRep = sqlite3_value_text(argv[2]); - if( zRep==0 ) return; - nRep = sqlite3_value_bytes(argv[2]); - assert( zRep==sqlite3_value_text(argv[2]) ); - nOut = nStr + 1; - assert( nOutaLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); - if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - sqlite3_free(zOut); - return; - } - zOld = zOut; - zOut = sqlite3_realloc(zOut, (int)nOut); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - sqlite3_free(zOld); - return; + /* Figure out how many bytes of space are required to store explicitly + ** specified collation sequence names. + */ + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( pExpr ){ + CollSeq *pColl = pExpr->pColl; + /* Either pColl!=0 or there was an OOM failure. But if an OOM + ** failure we have quit before reaching this point. */ + if( ALWAYS(pColl) ){ + nExtra += (1 + sqlite3Strlen30(pColl->zName)); } - memcpy(&zOut[j], zRep, nRep); - j += nRep; - i += nPattern-1; } } - assert( j+nStr-i+1==nOut ); - memcpy(&zOut[j], &zStr[i], nStr-i); - j += nStr - i; - assert( j<=nOut ); - zOut[j] = 0; - sqlite3_result_text(context, (char*)zOut, j, sqlite3_free); -} - -/* -** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. -** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. -*/ -static void trimFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zIn; /* Input string */ - const unsigned char *zCharSet; /* Set of characters to trim */ - int nIn; /* Number of bytes in input */ - int flags; /* 1: trimleft 2: trimright 3: trim */ - int i; /* Loop counter */ - unsigned char *aLen = 0; /* Length of each character in zCharSet */ - unsigned char **azChar = 0; /* Individual characters in zCharSet */ - int nChar; /* Number of characters in zCharSet */ - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ - return; + /* + ** Allocate the index structure. + */ + nName = sqlite3Strlen30(zName); + nCol = pList->nExpr; + pIndex = sqlite3DbMallocZero(db, + sizeof(Index) + /* Index structure */ + sizeof(int)*nCol + /* Index.aiColumn */ + sizeof(int)*(nCol+1) + /* Index.aiRowEst */ + sizeof(char *)*nCol + /* Index.azColl */ + sizeof(u8)*nCol + /* Index.aSortOrder */ + nName + 1 + /* Index.zName */ + nExtra /* Collation sequence names */ + ); + if( db->mallocFailed ){ + goto exit_create_index; } - zIn = sqlite3_value_text(argv[0]); - if( zIn==0 ) return; - nIn = sqlite3_value_bytes(argv[0]); - assert( zIn==sqlite3_value_text(argv[0]) ); - if( argc==1 ){ - static const unsigned char lenOne[] = { 1 }; - static unsigned char * const azOne[] = { (u8*)" " }; - nChar = 1; - aLen = (u8*)lenOne; - azChar = (unsigned char **)azOne; - zCharSet = 0; - }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ - return; + pIndex->azColl = (char**)(&pIndex[1]); + pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]); + pIndex->aiRowEst = (unsigned *)(&pIndex->aiColumn[nCol]); + pIndex->aSortOrder = (u8 *)(&pIndex->aiRowEst[nCol+1]); + pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]); + zExtra = (char *)(&pIndex->zName[nName+1]); + memcpy(pIndex->zName, zName, nName+1); + pIndex->pTable = pTab; + pIndex->nColumn = pList->nExpr; + pIndex->onError = (u8)onError; + pIndex->autoIndex = (u8)(pName==0); + pIndex->pSchema = db->aDb[iDb].pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + + /* Check to see if we should honor DESC requests on index columns + */ + if( pDb->pSchema->file_format>=4 ){ + sortOrderMask = -1; /* Honor DESC */ }else{ - const unsigned char *z; - for(z=zCharSet, nChar=0; *z; nChar++){ - SQLITE_SKIP_UTF8(z); - } - if( nChar>0 ){ - azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+1)); - if( azChar==0 ){ - return; - } - aLen = (unsigned char*)&azChar[nChar]; - for(z=zCharSet, nChar=0; *z; nChar++){ - azChar[nChar] = (unsigned char *)z; - SQLITE_SKIP_UTF8(z); - aLen[nChar] = (u8)(z - azChar[nChar]); - } - } + sortOrderMask = 0; /* Ignore DESC */ } - if( nChar>0 ){ - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); - if( flags & 1 ){ - while( nIn>0 ){ - int len = 0; - for(i=0; i=nChar ) break; - zIn += len; - nIn -= len; - } + + /* Scan the names of the columns of the table to be indexed and + ** load the column indices into the Index structure. Report an error + ** if any column is not found. + ** + ** TODO: Add a test to make sure that the same column is not named + ** more than once within the same index. Only the first instance of + ** the column will ever be used by the optimizer. Note that using the + ** same column more than once cannot be an error because that would + ** break backwards compatibility - it needs to be a warning. + */ + for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ + const char *zColName = pListItem->zName; + Column *pTabCol; + int requestedSortOrder; + char *zColl; /* Collation sequence name */ + + for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ + if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; } - if( flags & 2 ){ - while( nIn>0 ){ - int len = 0; - for(i=0; i=nChar ) break; - nIn -= len; + if( j>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "table %s has no column named %s", + pTab->zName, zColName); + pParse->checkSchema = 1; + goto exit_create_index; + } + pIndex->aiColumn[i] = j; + /* Justification of the ALWAYS(pListItem->pExpr->pColl): Because of + ** the way the "idxlist" non-terminal is constructed by the parser, + ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl + ** must exist or else there must have been an OOM error. But if there + ** was an OOM error, we would never reach this point. */ + if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){ + int nColl; + zColl = pListItem->pExpr->pColl->zName; + nColl = sqlite3Strlen30(zColl) + 1; + assert( nExtra>=nColl ); + memcpy(zExtra, zColl, nColl); + zColl = zExtra; + zExtra += nColl; + nExtra -= nColl; + }else{ + zColl = pTab->aCol[j].zColl; + if( !zColl ){ + zColl = db->pDfltColl->zName; } } - if( zCharSet ){ - sqlite3_free(azChar); + if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ + goto exit_create_index; } + pIndex->azColl[i] = zColl; + requestedSortOrder = pListItem->sortOrder & sortOrderMask; + pIndex->aSortOrder[i] = (u8)requestedSortOrder; } - sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); -} + sqlite3DefaultRowEst(pIndex); + if( pTab==pParse->pNewTable ){ + /* This routine has been called to create an automatic index as a + ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or + ** a PRIMARY KEY or UNIQUE clause following the column definitions. + ** i.e. one of: + ** + ** CREATE TABLE t(x PRIMARY KEY, y); + ** CREATE TABLE t(x, y, UNIQUE(x, y)); + ** + ** Either way, check to see if the table already has such an index. If + ** so, don't bother creating this one. This only applies to + ** automatically created indices. Users can do as they wish with + ** explicit indices. + ** + ** Two UNIQUE or PRIMARY KEY constraints are considered equivalent + ** (and thus suppressing the second one) even if they have different + ** sort orders. + ** + ** If there are different collating sequences or if the columns of + ** the constraint occur in different orders, then the constraints are + ** considered distinct and both result in separate indices. + */ + Index *pIdx; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int k; + assert( pIdx->onError!=OE_None ); + assert( pIdx->autoIndex ); + assert( pIndex->onError!=OE_None ); -/* IMP: R-25361-16150 This function is omitted from SQLite by default. It -** is only available if the SQLITE_SOUNDEX compile-time option is used -** when SQLite is built. -*/ -#ifdef SQLITE_SOUNDEX -/* -** Compute the soundex encoding of a word. -** -** IMP: R-59782-00072 The soundex(X) function returns a string that is the -** soundex encoding of the string X. -*/ -static void soundexFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - char zResult[8]; - const u8 *zIn; - int i, j; - static const unsigned char iCode[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, - 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, - 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, - }; - assert( argc==1 ); - zIn = (u8*)sqlite3_value_text(argv[0]); - if( zIn==0 ) zIn = (u8*)""; - for(i=0; zIn[i] && !sqlite3Isalpha(zIn[i]); i++){} - if( zIn[i] ){ - u8 prevcode = iCode[zIn[i]&0x7f]; - zResult[0] = sqlite3Toupper(zIn[i]); - for(j=1; j<4 && zIn[i]; i++){ - int code = iCode[zIn[i]&0x7f]; - if( code>0 ){ - if( code!=prevcode ){ - prevcode = code; - zResult[j++] = code + '0'; + if( pIdx->nColumn!=pIndex->nColumn ) continue; + for(k=0; knColumn; k++){ + const char *z1; + const char *z2; + if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; + z1 = pIdx->azColl[k]; + z2 = pIndex->azColl[k]; + if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; + } + if( k==pIdx->nColumn ){ + if( pIdx->onError!=pIndex->onError ){ + /* This constraint creates the same index as a previous + ** constraint specified somewhere in the CREATE TABLE statement. + ** However the ON CONFLICT clauses are different. If both this + ** constraint and the previous equivalent constraint have explicit + ** ON CONFLICT clauses this is an error. Otherwise, use the + ** explicitly specified behaviour for the index. + */ + if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ + sqlite3ErrorMsg(pParse, + "conflicting ON CONFLICT clauses specified", 0); + } + if( pIdx->onError==OE_Default ){ + pIdx->onError = pIndex->onError; + } } - }else{ - prevcode = 0; + goto exit_create_index; } } - while( j<4 ){ - zResult[j++] = '0'; + } + + /* Link the new Index structure to its table and to the other + ** in-memory database structures. + */ + if( db->init.busy ){ + Index *p; + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, sqlite3Strlen30(pIndex->zName), + pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + db->mallocFailed = 1; + goto exit_create_index; + } + db->flags |= SQLITE_InternChanges; + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; } - zResult[j] = 0; - sqlite3_result_text(context, zResult, 4, SQLITE_TRANSIENT); - }else{ - /* IMP: R-64894-50321 The string "?000" is returned if the argument - ** is NULL or contains no ASCII alphabetic characters. */ - sqlite3_result_text(context, "?000", 4, SQLITE_STATIC); } -} -#endif /* SQLITE_SOUNDEX */ -#ifndef SQLITE_OMIT_LOAD_EXTENSION -/* -** A function that loads a shared-library extension then returns NULL. -*/ -static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ - const char *zFile = (const char *)sqlite3_value_text(argv[0]); - const char *zProc; - sqlite3 *db = sqlite3_context_db_handle(context); - char *zErrMsg = 0; + /* If the db->init.busy is 0 then create the index on disk. This + ** involves writing the index into the master table and filling in the + ** index with the current table contents. + ** + ** The db->init.busy is 0 when the user first enters a CREATE INDEX + ** command. db->init.busy is 1 when a database is opened and + ** CREATE INDEX statements are read out of the master table. In + ** the latter case the index already exists on disk, which is why + ** we don't want to recreate it. + ** + ** If pTblName==0 it means this index is generated as a primary key + ** or UNIQUE constraint of a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. + */ + else{ /* if( db->init.busy==0 ) */ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; - if( argc==2 ){ - zProc = (const char *)sqlite3_value_text(argv[1]); - }else{ - zProc = 0; - } - if( zFile && sqlite3_load_extension(db, zFile, zProc, &zErrMsg) ){ - sqlite3_result_error(context, zErrMsg, -1); - sqlite3_free(zErrMsg); - } -} -#endif + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; -/* -** An instance of the following structure holds the context of a -** sum() or avg() aggregate computation. -*/ -typedef struct SumCtx SumCtx; -struct SumCtx { - double rSum; /* Floating point sum */ - i64 iSum; /* Integer sum */ - i64 cnt; /* Number of elements summed */ - u8 overflow; /* True if integer overflow seen */ - u8 approx; /* True if non-integer value was input to the sum */ -}; + /* Create the rootpage for the index + */ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); -/* -** Routines used to compute the sum, average, and total. -** -** The SUM() function follows the (broken) SQL standard which means -** that it returns NULL if it sums over no inputs. TOTAL returns -** 0.0 in that case. In addition, TOTAL always returns a float where -** SUM might return an integer if it never encounters a floating point -** value. TOTAL never fails, but SUM might through an exception if -** it overflows an integer. -*/ -static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ - SumCtx *p; - int type; - assert( argc==1 ); - UNUSED_PARAMETER(argc); - p = sqlite3_aggregate_context(context, sizeof(*p)); - type = sqlite3_value_numeric_type(argv[0]); - if( p && type!=SQLITE_NULL ){ - p->cnt++; - if( type==SQLITE_INTEGER ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum += v; - if( (p->approx|p->overflow)==0 ){ - i64 iNewSum = p->iSum + v; - int s1 = (int)(p->iSum >> (sizeof(i64)*8-1)); - int s2 = (int)(v >> (sizeof(i64)*8-1)); - int s3 = (int)(iNewSum >> (sizeof(i64)*8-1)); - p->overflow = ((s1&s2&~s3) | (~s1&~s2&s3))?1:0; - p->iSum = iNewSum; - } + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + if( pStart ){ + assert( pEnd!=0 ); + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", + pEnd->z - pName->z + 1, + pName->z); }else{ - p->rSum += sqlite3_value_double(argv[0]); - p->approx = 1; + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_master for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); + sqlite3VdbeAddOp1(v, OP_Expire, 0); } } -} -static void sumFinalize(sqlite3_context *context){ - SumCtx *p; - p = sqlite3_aggregate_context(context, 0); - if( p && p->cnt>0 ){ - if( p->overflow ){ - sqlite3_result_error(context,"integer overflow",-1); - }else if( p->approx ){ - sqlite3_result_double(context, p->rSum); + + /* When adding an index to the list of indices for a table, make + ** sure all indices labeled OE_Replace come after all those labeled + ** OE_Ignore. This is necessary for the correct constraint check + ** processing (in sqlite3GenerateConstraintChecks()) as part of + ** UPDATE and INSERT statements. + */ + if( db->init.busy || pTblName==0 ){ + if( onError!=OE_Replace || pTab->pIndex==0 + || pTab->pIndex->onError==OE_Replace){ + pIndex->pNext = pTab->pIndex; + pTab->pIndex = pIndex; }else{ - sqlite3_result_int64(context, p->iSum); + Index *pOther = pTab->pIndex; + while( pOther->pNext && pOther->pNext->onError!=OE_Replace ){ + pOther = pOther->pNext; + } + pIndex->pNext = pOther->pNext; + pOther->pNext = pIndex; } + pRet = pIndex; + pIndex = 0; } -} -static void avgFinalize(sqlite3_context *context){ - SumCtx *p; - p = sqlite3_aggregate_context(context, 0); - if( p && p->cnt>0 ){ - sqlite3_result_double(context, p->rSum/(double)p->cnt); + + /* Clean up before exiting */ +exit_create_index: + if( pIndex ){ + sqlite3DbFree(db, pIndex->zColAff); + sqlite3DbFree(db, pIndex); } + sqlite3ExprListDelete(db, pList); + sqlite3SrcListDelete(db, pTblName); + sqlite3DbFree(db, zName); + return pRet; } -static void totalFinalize(sqlite3_context *context){ - SumCtx *p; - p = sqlite3_aggregate_context(context, 0); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - sqlite3_result_double(context, p ? p->rSum : (double)0); -} - -/* -** The following structure keeps track of state information for the -** count() aggregate function. -*/ -typedef struct CountCtx CountCtx; -struct CountCtx { - i64 n; -}; /* -** Routines to implement the count() aggregate function. +** Fill the Index.aiRowEst[] array with default information - information +** to be used when we have not run the ANALYZE command. +** +** aiRowEst[0] is suppose to contain the number of elements in the index. +** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the +** number of rows in the table that match any particular value of the +** first column of the index. aiRowEst[2] is an estimate of the number +** of rows that match any particular combiniation of the first 2 columns +** of the index. And so forth. It must always be the case that +* +** aiRowEst[N]<=aiRowEst[N-1] +** aiRowEst[N]>=1 +** +** Apart from that, we have little to go on besides intuition as to +** how aiRowEst[] should be initialized. The numbers generated here +** are based on typical values found in actual indices. */ -static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ - CountCtx *p; - p = sqlite3_aggregate_context(context, sizeof(*p)); - if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){ - p->n++; +SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ + unsigned *a = pIdx->aiRowEst; + int i; + unsigned n; + assert( a!=0 ); + a[0] = pIdx->pTable->nRowEst; + if( a[0]<10 ) a[0] = 10; + n = 10; + for(i=1; i<=pIdx->nColumn; i++){ + a[i] = n; + if( n>5 ) n--; + } + if( pIdx->onError!=OE_None ){ + a[pIdx->nColumn] = 1; } - -#ifndef SQLITE_OMIT_DEPRECATED - /* The sqlite3_aggregate_count() function is deprecated. But just to make - ** sure it still operates correctly, verify that its count agrees with our - ** internal count when using count(*) and when the total count can be - ** expressed as a 32-bit integer. */ - assert( argc==1 || p==0 || p->n>0x7fffffff - || p->n==sqlite3_aggregate_count(context) ); -#endif -} -static void countFinalize(sqlite3_context *context){ - CountCtx *p; - p = sqlite3_aggregate_context(context, 0); - sqlite3_result_int64(context, p ? p->n : 0); } /* -** Routines to implement min() and max() aggregate functions. +** This routine will drop an existing named index. This routine +** implements the DROP INDEX statement. */ -static void minmaxStep( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - Mem *pArg = (Mem *)argv[0]; - Mem *pBest; - UNUSED_PARAMETER(NotUsed); - - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); - if( !pBest ) return; +SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ + Index *pIndex; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; - if( pBest->flags ){ - int max; - int cmp; - CollSeq *pColl = sqlite3GetFuncCollSeq(context); - /* This step function is used for both the min() and max() aggregates, - ** the only difference between the two being that the sense of the - ** comparison is inverted. For the max() aggregate, the - ** sqlite3_user_data() function returns (void *)-1. For min() it - ** returns (void *)db, where db is the sqlite3* database pointer. - ** Therefore the next statement sets variable 'max' to 1 for the max() - ** aggregate, or 0 for min(). - */ - max = sqlite3_user_data(context)!=0; - cmp = sqlite3MemCompare(pBest, pArg, pColl); - if( (max && cmp<0) || (!max && cmp>0) ){ - sqlite3VdbeMemCopy(pBest, pArg); + assert( pParse->nErr==0 ); /* Never called with prior errors */ + if( db->mallocFailed ){ + goto exit_drop_index; + } + assert( pName->nSrc==1 ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_drop_index; + } + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + if( pIndex==0 ){ + if( !ifExists ){ + sqlite3ErrorMsg(pParse, "no such index: %S", pName, 0); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); } - }else{ - sqlite3VdbeMemCopy(pBest, pArg); + pParse->checkSchema = 1; + goto exit_drop_index; } -} -static void minMaxFinalize(sqlite3_context *context){ - sqlite3_value *pRes; - pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); - if( pRes ){ - if( ALWAYS(pRes->flags) ){ - sqlite3_result_value(context, pRes); + if( pIndex->autoIndex ){ + sqlite3ErrorMsg(pParse, "index associated with UNIQUE " + "or PRIMARY KEY constraint cannot be dropped", 0); + goto exit_drop_index; + } + iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_DROP_INDEX; + Table *pTab = pIndex->pTable; + const char *zDb = db->aDb[iDb].zName; + const char *zTab = SCHEMA_TABLE(iDb); + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + goto exit_drop_index; } - sqlite3VdbeMemRelease(pRes); + if( !OMIT_TEMPDB && iDb ) code = SQLITE_DROP_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, code, pIndex->zName, pTab->zName, zDb) ){ + goto exit_drop_index; + } + } +#endif + + /* Generate code to remove the index and from the master table */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pIndex->zName + ); + if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_stat1 WHERE idx=%Q", + db->aDb[iDb].zName, pIndex->zName + ); + } + sqlite3ChangeCookie(pParse, iDb); + destroyRootPage(pParse, pIndex->tnum, iDb); + sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); } + +exit_drop_index: + sqlite3SrcListDelete(db, pName); } /* -** group_concat(EXPR, ?SEPARATOR?) +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine allocates a new +** object on the end of the array. +** +** *pnEntry is the number of entries already in use. *pnAlloc is +** the previously allocated size of the array. initSize is the +** suggested initial array size allocation. +** +** The index of the new entry is returned in *pIdx. +** +** This routine returns a pointer to the array of objects. This +** might be the same as the pArray parameter or it might be a different +** pointer if the array was resized. */ -static void groupConcatStep( - sqlite3_context *context, - int argc, - sqlite3_value **argv +SQLITE_PRIVATE void *sqlite3ArrayAllocate( + sqlite3 *db, /* Connection to notify of malloc failures */ + void *pArray, /* Array of objects. Might be reallocated */ + int szEntry, /* Size of each object in the array */ + int initSize, /* Suggested initial allocation, in elements */ + int *pnEntry, /* Number of objects currently in use */ + int *pnAlloc, /* Current size of the allocation, in elements */ + int *pIdx /* Write the index of a new slot here */ ){ - const char *zVal; - StrAccum *pAccum; - const char *zSep; - int nVal, nSep; - assert( argc==1 || argc==2 ); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); - - if( pAccum ){ - sqlite3 *db = sqlite3_context_db_handle(context); - int firstTerm = pAccum->useMalloc==0; - pAccum->useMalloc = 2; - pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; - if( !firstTerm ){ - if( argc==2 ){ - zSep = (char*)sqlite3_value_text(argv[1]); - nSep = sqlite3_value_bytes(argv[1]); - }else{ - zSep = ","; - nSep = 1; - } - sqlite3StrAccumAppend(pAccum, zSep, nSep); - } - zVal = (char*)sqlite3_value_text(argv[0]); - nVal = sqlite3_value_bytes(argv[0]); - sqlite3StrAccumAppend(pAccum, zVal, nVal); - } -} -static void groupConcatFinalize(sqlite3_context *context){ - StrAccum *pAccum; - pAccum = sqlite3_aggregate_context(context, 0); - if( pAccum ){ - if( pAccum->tooBig ){ - sqlite3_result_error_toobig(context); - }else if( pAccum->mallocFailed ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, - sqlite3_free); + char *z; + if( *pnEntry >= *pnAlloc ){ + void *pNew; + int newSize; + newSize = (*pnAlloc)*2 + initSize; + pNew = sqlite3DbRealloc(db, pArray, newSize*szEntry); + if( pNew==0 ){ + *pIdx = -1; + return pArray; } + *pnAlloc = sqlite3DbMallocSize(db, pNew)/szEntry; + pArray = pNew; } + z = (char*)pArray; + memset(&z[*pnEntry * szEntry], 0, szEntry); + *pIdx = *pnEntry; + ++*pnEntry; + return pArray; } /* -** This routine does per-connection function registration. Most -** of the built-in functions above are part of the global function set. -** This routine only deals with those that are not global. +** Append a new element to the given IdList. Create a new IdList if +** need be. +** +** A new IdList is returned, or NULL if malloc() fails. */ -SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ - int rc = sqlite3_overload_function(db, "MATCH", 2); - assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); - if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; +SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ + int i; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + if( pList==0 ) return 0; + pList->nAlloc = 0; } + pList->a = sqlite3ArrayAllocate( + db, + pList->a, + sizeof(pList->a[0]), + 5, + &pList->nId, + &pList->nAlloc, + &i + ); + if( i<0 ){ + sqlite3IdListDelete(db, pList); + return 0; + } + pList->a[i].zName = sqlite3NameFromToken(db, pToken); + return pList; } /* -** Set the LIKEOPT flag on the 2-argument function with the given name. +** Delete an IdList. */ -static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ - FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), - 2, SQLITE_UTF8, 0); - if( ALWAYS(pDef) ){ - pDef->flags = flagVal; +SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ + int i; + if( pList==0 ) return; + for(i=0; inId; i++){ + sqlite3DbFree(db, pList->a[i].zName); } + sqlite3DbFree(db, pList->a); + sqlite3DbFree(db, pList); } /* -** Register the built-in LIKE and GLOB functions. The caseSensitive -** parameter determines whether or not the LIKE operator is case -** sensitive. GLOB is always case sensitive. +** Return the index in pList of the identifier named zId. Return -1 +** if not found. */ -SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ - struct compareInfo *pInfo; - if( caseSensitive ){ - pInfo = (struct compareInfo*)&likeInfoAlt; - }else{ - pInfo = (struct compareInfo*)&likeInfoNorm; +SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ + int i; + if( pList==0 ) return -1; + for(i=0; inId; i++){ + if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; } - sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "glob", 2, SQLITE_ANY, - (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); - setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); - setLikeOptFlag(db, "like", - caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); + return -1; } /* -** pExpr points to an expression which implements a function. If -** it is appropriate to apply the LIKE optimization to that function -** then set aWc[0] through aWc[2] to the wildcard characters and -** return TRUE. If the function is not a LIKE-style function then -** return FALSE. +** Expand the space allocated for the given SrcList object by +** creating nExtra new slots beginning at iStart. iStart is zero based. +** New slots are zeroed. +** +** For example, suppose a SrcList initially contains two entries: A,B. +** To append 3 new entries onto the end, do this: +** +** sqlite3SrcListEnlarge(db, pSrclist, 3, 2); +** +** After the call above it would contain: A, B, nil, nil, nil. +** If the iStart argument had been 1 instead of 2, then the result +** would have been: A, nil, nil, nil, B. To prepend the new slots, +** the iStart value would be 0. The result then would +** be: nil, nil, nil, A, B. +** +** If a memory allocation fails the SrcList is unchanged. The +** db->mallocFailed flag will be set to true. */ -SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ - FuncDef *pDef; - if( pExpr->op!=TK_FUNCTION - || !pExpr->x.pList - || pExpr->x.pList->nExpr!=2 - ){ - return 0; +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( + sqlite3 *db, /* Database connection to notify of OOM errors */ + SrcList *pSrc, /* The SrcList to be enlarged */ + int nExtra, /* Number of new slots to add to pSrc->a[] */ + int iStart /* Index in pSrc->a[] of first new slot */ +){ + int i; + + /* Sanity checking on calling parameters */ + assert( iStart>=0 ); + assert( nExtra>=1 ); + assert( pSrc!=0 ); + assert( iStart<=pSrc->nSrc ); + + /* Allocate additional space if needed */ + if( pSrc->nSrc+nExtra>pSrc->nAlloc ){ + SrcList *pNew; + int nAlloc = pSrc->nSrc+nExtra; + int nGot; + pNew = sqlite3DbRealloc(db, pSrc, + sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + if( pNew==0 ){ + assert( db->mallocFailed ); + return pSrc; + } + pSrc = pNew; + nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; + pSrc->nAlloc = (u16)nGot; } - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, - sqlite3Strlen30(pExpr->u.zToken), - 2, SQLITE_UTF8, 0); - if( NEVER(pDef==0) || (pDef->flags & SQLITE_FUNC_LIKE)==0 ){ - return 0; + + /* Move existing slots that come after the newly inserted slots + ** out of the way */ + for(i=pSrc->nSrc-1; i>=iStart; i--){ + pSrc->a[i+nExtra] = pSrc->a[i]; } + pSrc->nSrc += (i16)nExtra; - /* The memcpy() statement assumes that the wildcard characters are - ** the first three statements in the compareInfo structure. The - ** asserts() that follow verify that assumption - */ - memcpy(aWc, pDef->pUserData, 3); - assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); - assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); - assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); - *pIsNocase = (pDef->flags & SQLITE_FUNC_CASE)==0; - return 1; + /* Zero the newly allocated slots */ + memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); + for(i=iStart; ia[i].iCursor = -1; + } + + /* Return a pointer to the enlarged SrcList */ + return pSrc; } + /* -** All all of the FuncDef structures in the aBuiltinFunc[] array above -** to the global function hash table. This occurs at start-time (as -** a consequence of calling sqlite3_initialize()). +** Append a new table name to the given SrcList. Create a new SrcList if +** need be. A new entry is created in the SrcList even if pTable is NULL. ** -** After this routine runs -*/ -SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ - /* - ** The following array holds FuncDef structures for all of the functions - ** defined in this file. - ** - ** The array cannot be constant since changes are made to the - ** FuncDef.pHash elements at start-time. The elements of this array - ** are read-only after initialization is complete. - */ - static SQLITE_WSD FuncDef aBuiltinFunc[] = { - FUNCTION(ltrim, 1, 1, 0, trimFunc ), - FUNCTION(ltrim, 2, 1, 0, trimFunc ), - FUNCTION(rtrim, 1, 2, 0, trimFunc ), - FUNCTION(rtrim, 2, 2, 0, trimFunc ), - FUNCTION(trim, 1, 3, 0, trimFunc ), - FUNCTION(trim, 2, 3, 0, trimFunc ), - FUNCTION(min, -1, 0, 1, minmaxFunc ), - FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), - FUNCTION(max, -1, 1, 1, minmaxFunc ), - FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), - FUNCTION(substr, 2, 0, 0, substrFunc ), - FUNCTION(substr, 3, 0, 0, substrFunc ), - FUNCTION(abs, 1, 0, 0, absFunc ), -#ifndef SQLITE_OMIT_FLOATING_POINT - FUNCTION(round, 1, 0, 0, roundFunc ), - FUNCTION(round, 2, 0, 0, roundFunc ), -#endif - FUNCTION(upper, 1, 0, 0, upperFunc ), - FUNCTION(lower, 1, 0, 0, lowerFunc ), - FUNCTION(coalesce, 1, 0, 0, 0 ), - FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, - FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, - FUNCTION(random, 0, 0, 0, randomFunc ), - FUNCTION(randomblob, 1, 0, 0, randomBlob ), - FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ - FUNCTION(quote, 1, 0, 0, quoteFunc ), - FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), - FUNCTION(changes, 0, 0, 0, changes ), - FUNCTION(total_changes, 0, 0, 0, total_changes ), - FUNCTION(replace, 3, 0, 0, replaceFunc ), - FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), - #ifdef SQLITE_SOUNDEX - FUNCTION(soundex, 1, 0, 0, soundexFunc ), - #endif - #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), - #endif - AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), - AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), - AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, - AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), - AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), - AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), - - LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #ifdef SQLITE_CASE_SENSITIVE_LIKE - LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #else - LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), - LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), - #endif - }; - - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); - - for(i=0; inAlloc = 1; + } + pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); + if( db->mallocFailed ){ + sqlite3SrcListDelete(db, pList); + return 0; + } + pItem = &pList->a[pList->nSrc-1]; + if( pDatabase && pDatabase->z==0 ){ + pDatabase = 0; + } + if( pDatabase ){ + Token *pTemp = pDatabase; + pDatabase = pTable; + pTable = pTemp; + } + pItem->zName = sqlite3NameFromToken(db, pTable); + pItem->zDatabase = sqlite3NameFromToken(db, pDatabase); + return pList; +} /* -** VDBE Calling Convention -** ----------------------- -** -** Example: -** -** For the following INSERT statement: -** -** CREATE TABLE t1(a, b INTEGER PRIMARY KEY, c); -** INSERT INTO t1 VALUES(1, 2, 3.1); -** -** Register (x): 2 (type integer) -** Register (x+1): 1 (type integer) -** Register (x+2): NULL (type NULL) -** Register (x+3): 3.1 (type real) +** Assign VdbeCursor index numbers to all tables in a SrcList */ +SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ + int i; + struct SrcList_item *pItem; + assert(pList || pParse->db->mallocFailed ); + if( pList ){ + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pItem->iCursor>=0 ) break; + pItem->iCursor = pParse->nTab++; + if( pItem->pSelect ){ + sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + } + } + } +} /* -** A foreign key constraint requires that the key columns in the parent -** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. -** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema a unique index on the parent key columns. -** -** If successful, zero is returned. If the parent key is an INTEGER PRIMARY -** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx -** is set to point to the unique index. -** -** If the parent key consists of a single column (the foreign key constraint -** is not a composite foreign key), output variable *paiCol is set to NULL. -** Otherwise, it is set to point to an allocated array of size N, where -** N is the number of columns in the parent key. The first element of the -** array is the index of the child table column that is mapped by the FK -** constraint to the parent table column stored in the left-most column -** of index *ppIdx. The second element of the array is the index of the -** child table column that corresponds to the second left-most column of -** *ppIdx, and so on. -** -** If the required index cannot be found, either because: -** -** 1) The named parent key columns do not exist, or -** -** 2) The named parent key columns do exist, but are not subject to a -** UNIQUE or PRIMARY KEY constraint, or -** -** 3) No parent key columns were provided explicitly as part of the -** foreign key definition, and the parent table does not have a -** PRIMARY KEY, or -** -** 4) No parent key columns were provided explicitly as part of the -** foreign key definition, and the PRIMARY KEY of the parent table -** consists of a a different number of columns to the child key in -** the child table. +** Delete an entire SrcList including all its substructure. +*/ +SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ + int i; + struct SrcList_item *pItem; + if( pList==0 ) return; + for(pItem=pList->a, i=0; inSrc; i++, pItem++){ + sqlite3DbFree(db, pItem->zDatabase); + sqlite3DbFree(db, pItem->zName); + sqlite3DbFree(db, pItem->zAlias); + sqlite3DbFree(db, pItem->zIndex); + sqlite3DeleteTable(db, pItem->pTab); + sqlite3SelectDelete(db, pItem->pSelect); + sqlite3ExprDelete(db, pItem->pOn); + sqlite3IdListDelete(db, pItem->pUsing); + } + sqlite3DbFree(db, pList); +} + +/* +** This routine is called by the parser to add a new term to the +** end of a growing FROM clause. The "p" parameter is the part of +** the FROM clause that has already been constructed. "p" is NULL +** if this is the first term of the FROM clause. pTable and pDatabase +** are the name of the table and database named in the FROM clause term. +** pDatabase is NULL if the database name qualifier is missing - the +** usual case. If the term has a alias, then pAlias points to the +** alias token. If the term is a subquery, then pSubquery is the +** SELECT statement that the subquery encodes. The pTable and +** pDatabase parameters are NULL for subqueries. The pOn and pUsing +** parameters are the content of the ON and USING clauses. ** -** then non-zero is returned, and a "foreign key mismatch" error loaded -** into pParse. If an OOM error occurs, non-zero is returned and the -** pParse->db->mallocFailed flag is set. +** Return a new SrcList which encodes is the FROM with the new +** term added. */ -static int locateFkeyIndex( - Parse *pParse, /* Parse context to store any error in */ - Table *pParent, /* Parent table of FK constraint pFKey */ - FKey *pFKey, /* Foreign key to find index for */ - Index **ppIdx, /* OUT: Unique index on parent table */ - int **paiCol /* OUT: Map of index columns in pFKey */ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( + Parse *pParse, /* Parsing context */ + SrcList *p, /* The left part of the FROM clause already seen */ + Token *pTable, /* Name of the table to add to the FROM clause */ + Token *pDatabase, /* Name of the database containing pTable */ + Token *pAlias, /* The right-hand side of the AS subexpression */ + Select *pSubquery, /* A subquery used in place of a table name */ + Expr *pOn, /* The ON clause of a join */ + IdList *pUsing /* The USING clause of a join */ ){ - Index *pIdx = 0; /* Value to return via *ppIdx */ - int *aiCol = 0; /* Value to return via *paiCol */ - int nCol = pFKey->nCol; /* Number of columns in parent key */ - char *zKey = pFKey->aCol[0].zCol; /* Name of left-most parent key column */ - - /* The caller is responsible for zeroing output parameters. */ - assert( ppIdx && *ppIdx==0 ); - assert( !paiCol || *paiCol==0 ); - assert( pParse ); - - /* If this is a non-composite (single column) foreign key, check if it - ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx - ** and *paiCol set to zero and return early. - ** - ** Otherwise, for a composite foreign key (more than one column), allocate - ** space for the aiCol array (returned via output parameter *paiCol). - ** Non-composite foreign keys do not require the aiCol array. - */ - if( nCol==1 ){ - /* The FK maps to the IPK if any of the following are true: - ** - ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly - ** mapped to the primary key of table pParent, or - ** 2) The FK is explicitly mapped to a column declared as INTEGER - ** PRIMARY KEY. - */ - if( pParent->iPKey>=0 ){ - if( !zKey ) return 0; - if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zName, zKey) ) return 0; - } - }else if( paiCol ){ - assert( nCol>1 ); - aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); - if( !aiCol ) return 1; - *paiCol = aiCol; + struct SrcList_item *pItem; + sqlite3 *db = pParse->db; + if( !p && (pOn || pUsing) ){ + sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", + (pOn ? "ON" : "USING") + ); + goto append_from_error; } - - for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->nColumn==nCol && pIdx->onError!=OE_None ){ - /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number - ** of columns. If each indexed column corresponds to a foreign key - ** column of pFKey, then this index is a winner. */ - - if( zKey==0 ){ - /* If zKey is NULL, then this foreign key is implicitly mapped to - ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be - ** identified by the test (Index.autoIndex==2). */ - if( pIdx->autoIndex==2 ){ - if( aiCol ){ - int i; - for(i=0; iaCol[i].iFrom; - } - break; - } - }else{ - /* If zKey is non-NULL, then this foreign key was declared to - ** map to an explicit list of columns in table pParent. Check if this - ** index matches those columns. Also, check that the index uses - ** the default collation sequences for each column. */ - int i, j; - for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ - char *zDfltColl; /* Def. collation for column */ - char *zIdxCol; /* Name of indexed column */ - - /* If the index uses a collation sequence that is different from - ** the default collation sequence for the column, this index is - ** unusable. Bail out early in this case. */ - zDfltColl = pParent->aCol[iCol].zColl; - if( !zDfltColl ){ - zDfltColl = "BINARY"; - } - if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; - - zIdxCol = pParent->aCol[iCol].zName; - for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ - if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; - break; - } - } - if( j==nCol ) break; - } - if( i==nCol ) break; /* pIdx is usable */ - } - } + p = sqlite3SrcListAppend(db, p, pTable, pDatabase); + if( p==0 || NEVER(p->nSrc==0) ){ + goto append_from_error; } - - if( !pIdx ){ - if( !pParse->disableTriggers ){ - sqlite3ErrorMsg(pParse, "foreign key mismatch"); - } - sqlite3DbFree(pParse->db, aiCol); - return 1; + pItem = &p->a[p->nSrc-1]; + assert( pAlias!=0 ); + if( pAlias->n ){ + pItem->zAlias = sqlite3NameFromToken(db, pAlias); } + pItem->pSelect = pSubquery; + pItem->pOn = pOn; + pItem->pUsing = pUsing; + return p; - *ppIdx = pIdx; + append_from_error: + assert( p==0 ); + sqlite3ExprDelete(db, pOn); + sqlite3IdListDelete(db, pUsing); + sqlite3SelectDelete(db, pSubquery); return 0; } /* -** This function is called when a row is inserted into or deleted from the -** child table of foreign key constraint pFKey. If an SQL UPDATE is executed -** on the child table of pFKey, this function is invoked twice for each row -** affected - once to "delete" the old row, and then again to "insert" the -** new row. +** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** element of the source-list passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ + assert( pIndexedBy!=0 ); + if( p && ALWAYS(p->nSrc>0) ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->notIndexed==0 && pItem->zIndex==0 ); + if( pIndexedBy->n==1 && !pIndexedBy->z ){ + /* A "NOT INDEXED" clause was supplied. See parse.y + ** construct "indexed_opt" for details. */ + pItem->notIndexed = 1; + }else{ + pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); + } + } +} + +/* +** When building up a FROM clause in the parser, the join operator +** is initially attached to the left operand. But the code generator +** expects the join operator to be on the right operand. This routine +** Shifts all join operators from left to right for an entire FROM +** clause. ** -** Each time it is called, this function generates VDBE code to locate the -** row in the parent table that corresponds to the row being inserted into -** or deleted from the child table. If the parent row can be found, no -** special action is taken. Otherwise, if the parent row can *not* be -** found in the parent table: +** Example: Suppose the join is like this: ** -** Operation | FK type | Action taken -** -------------------------------------------------------------------------- -** INSERT immediate Increment the "immediate constraint counter". -** -** DELETE immediate Decrement the "immediate constraint counter". -** -** INSERT deferred Increment the "deferred constraint counter". -** -** DELETE deferred Decrement the "deferred constraint counter". +** A natural cross join B ** -** These operations are identified in the comment at the top of this file -** (fkey.c) as "I.1" and "D.1". +** The operator is "natural cross join". The A and B operands are stored +** in p->a[0] and p->a[1], respectively. The parser initially stores the +** operator with A. This routine shifts that operator over to B. */ -static void fkLookupParent( - Parse *pParse, /* Parse context */ - int iDb, /* Index of database housing pTab */ - Table *pTab, /* Parent table of FK pFKey */ - Index *pIdx, /* Unique index on parent key columns in pTab */ - FKey *pFKey, /* Foreign key constraint */ - int *aiCol, /* Map from parent key columns to child table columns */ - int regData, /* Address of array containing child table row */ - int nIncr, /* Increment constraint counter by this */ - int isIgnore /* If true, pretend pTab contains all NULL values */ -){ - int i; /* Iterator variable */ - Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ - int iCur = pParse->nTab - 1; /* Cursor number to use */ - int iOk = sqlite3VdbeMakeLabel(v); /* jump here if parent key found */ - - /* If nIncr is less than zero, then check at runtime if there are any - ** outstanding constraints to resolve. If there are not, there is no need - ** to check if deleting this row resolves any outstanding violations. - ** - ** Check if any of the key columns in the child table row are NULL. If - ** any are, then the constraint is considered satisfied. No need to - ** search for a matching row in the parent table. */ - if( nIncr<0 ){ - sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); - } - for(i=0; inCol; i++){ - int iReg = aiCol[i] + regData + 1; - sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); - } - - if( isIgnore==0 ){ - if( pIdx==0 ){ - /* If pIdx is NULL, then the parent key is the INTEGER PRIMARY KEY - ** column of the parent table (table pTab). */ - int iMustBeInt; /* Address of MustBeInt instruction */ - int regTemp = sqlite3GetTempReg(pParse); - - /* Invoke MustBeInt to coerce the child key value to an integer (i.e. - ** apply the affinity of the parent key). If this fails, then there - ** is no matching parent key. Before using MustBeInt, make a copy of - ** the value. Otherwise, the value inserted into the child key column - ** will have INTEGER affinity applied to it, which may not be correct. */ - sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); - iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); - - /* If the parent table is the same as the child table, and we are about - ** to increment the constraint-counter (i.e. this is an INSERT operation), - ** then check if the row being inserted matches itself. If so, do not - ** increment the constraint-counter. */ - if( pTab==pFKey->pFrom && nIncr==1 ){ - sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); - } - - sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); - sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); - sqlite3VdbeJumpHere(v, iMustBeInt); - sqlite3ReleaseTempReg(pParse, regTemp); - }else{ - int nCol = pFKey->nCol; - int regTemp = sqlite3GetTempRange(pParse, nCol); - int regRec = sqlite3GetTempReg(pParse); - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); - - sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); - sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); - for(i=0; ipFrom && nIncr==1 ){ - int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; - for(i=0; iaiColumn[i]+1+regData; - sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); - } - - sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0); - sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); - - sqlite3ReleaseTempReg(pParse, regRec); - sqlite3ReleaseTempRange(pParse, regTemp, nCol); - } - } - - if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){ - /* Special case: If this is an INSERT statement that will insert exactly - ** one row into the table, raise a constraint immediately instead of - ** incrementing a counter. This is necessary as the VM code is being - ** generated for will not open a statement transaction. */ - assert( nIncr==1 ); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC - ); - }else{ - if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ + if( p && p->a ){ + int i; + for(i=p->nSrc-1; i>0; i--){ + p->a[i].jointype = p->a[i-1].jointype; } - sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + p->a[0].jointype = 0; } - - sqlite3VdbeResolveLabel(v, iOk); - sqlite3VdbeAddOp1(v, OP_Close, iCur); } /* -** This function is called to generate code executed when a row is deleted -** from the parent table of foreign key constraint pFKey and, if pFKey is -** deferred, when a row is inserted into the same table. When generating -** code for an SQL UPDATE operation, this function may be called twice - -** once to "delete" the old row and once to "insert" the new row. -** -** The code generated by this function scans through the rows in the child -** table that correspond to the parent table row being deleted or inserted. -** For each child row found, one of the following actions is taken: -** -** Operation | FK type | Action taken -** -------------------------------------------------------------------------- -** DELETE immediate Increment the "immediate constraint counter". -** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "foreign key constraint failed" exception. -** -** INSERT immediate Decrement the "immediate constraint counter". -** -** DELETE deferred Increment the "deferred constraint counter". -** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "foreign key constraint failed" exception. -** -** INSERT deferred Decrement the "deferred constraint counter". -** -** These operations are identified in the comment at the top of this file -** (fkey.c) as "I.2" and "D.2". +** Begin a transaction */ -static void fkScanChildren( - Parse *pParse, /* Parse context */ - SrcList *pSrc, /* SrcList containing the table to scan */ - Table *pTab, - Index *pIdx, /* Foreign key index */ - FKey *pFKey, /* Foreign key relationship */ - int *aiCol, /* Map from pIdx cols to child table cols */ - int regData, /* Referenced table data starts here */ - int nIncr /* Amount to increment deferred counter by */ -){ - sqlite3 *db = pParse->db; /* Database handle */ - int i; /* Iterator variable */ - Expr *pWhere = 0; /* WHERE clause to scan with */ - NameContext sNameContext; /* Context used to resolve WHERE clause */ - WhereInfo *pWInfo; /* Context used by sqlite3WhereXXX() */ - int iFkIfZero = 0; /* Address of OP_FkIfZero */ - Vdbe *v = sqlite3GetVdbe(pParse); - - assert( !pIdx || pIdx->pTable==pTab ); - - if( nIncr<0 ){ - iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); - } - - /* Create an Expr object representing an SQL expression like: - ** - ** = AND = ... - ** - ** The collation sequence used for the comparison should be that of - ** the parent key columns. The affinity of the parent key column should - ** be applied to each child key value before the comparison takes place. - */ - for(i=0; inCol; i++){ - Expr *pLeft; /* Value from parent table row */ - Expr *pRight; /* Column ref to child table */ - Expr *pEq; /* Expression (pLeft = pRight) */ - int iCol; /* Index of column in child table */ - const char *zCol; /* Name of column in child table */ +SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ + sqlite3 *db; + Vdbe *v; + int i; - pLeft = sqlite3Expr(db, TK_REGISTER, 0); - if( pLeft ){ - /* Set the collation sequence and affinity of the LHS of each TK_EQ - ** expression to the parent key column defaults. */ - if( pIdx ){ - Column *pCol; - iCol = pIdx->aiColumn[i]; - pCol = &pTab->aCol[iCol]; - if( pTab->iPKey==iCol ) iCol = -1; - pLeft->iTable = regData+iCol+1; - pLeft->affinity = pCol->affinity; - pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); - }else{ - pLeft->iTable = regData; - pLeft->affinity = SQLITE_AFF_INTEGER; - } - } - iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; - assert( iCol>=0 ); - zCol = pFKey->pFrom->aCol[iCol].zName; - pRight = sqlite3Expr(db, TK_ID, zCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + assert( pParse!=0 ); + db = pParse->db; + assert( db!=0 ); +/* if( db->aDb[0].pBt==0 ) return; */ + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ + return; } - - /* If the child table is the same as the parent table, and this scan - ** is taking place as part of a DELETE operation (operation D.2), omit the - ** row being deleted from the scan by adding ($rowid != rowid) to the WHERE - ** clause, where $rowid is the rowid of the row being deleted. */ - if( pTab==pFKey->pFrom && nIncr>0 ){ - Expr *pEq; /* Expression (pLeft = pRight) */ - Expr *pLeft; /* Value from parent table row */ - Expr *pRight; /* Column ref to child table */ - pLeft = sqlite3Expr(db, TK_REGISTER, 0); - pRight = sqlite3Expr(db, TK_COLUMN, 0); - if( pLeft && pRight ){ - pLeft->iTable = regData; - pLeft->affinity = SQLITE_AFF_INTEGER; - pRight->iTable = pSrc->a[0].iCursor; - pRight->iColumn = -1; + v = sqlite3GetVdbe(pParse); + if( !v ) return; + if( type!=TK_DEFERRED ){ + for(i=0; inDb; i++){ + sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); + sqlite3VdbeUsesBtree(v, i); } - pEq = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); } + sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); +} - /* Resolve the references in the WHERE clause. */ - memset(&sNameContext, 0, sizeof(NameContext)); - sNameContext.pSrcList = pSrc; - sNameContext.pParse = pParse; - sqlite3ResolveExprNames(&sNameContext, pWhere); +/* +** Commit a transaction +*/ +SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ + sqlite3 *db; + Vdbe *v; - /* Create VDBE to loop through the entries in pSrc that match the WHERE - ** clause. If the constraint is not deferred, throw an exception for - ** each row found. Otherwise, for deferred constraints, increment the - ** deferred constraint counter by nIncr for each row selected. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); - if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; + assert( pParse!=0 ); + db = pParse->db; + assert( db!=0 ); +/* if( db->aDb[0].pBt==0 ) return; */ + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ){ + return; } - sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); - if( pWInfo ){ - sqlite3WhereEnd(pWInfo); + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); } +} - /* Clean up the WHERE clause constructed above. */ - sqlite3ExprDelete(db, pWhere); - if( iFkIfZero ){ - sqlite3VdbeJumpHere(v, iFkIfZero); +/* +** Rollback a transaction +*/ +SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ + sqlite3 *db; + Vdbe *v; + + assert( pParse!=0 ); + db = pParse->db; + assert( db!=0 ); +/* if( db->aDb[0].pBt==0 ) return; */ + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 1); } } /* -** This function returns a pointer to the head of a linked list of FK -** constraints for which table pTab is the parent table. For example, -** given the following schema: -** -** CREATE TABLE t1(a PRIMARY KEY); -** CREATE TABLE t2(b REFERENCES t1(a); -** -** Calling this function with table "t1" as an argument returns a pointer -** to the FKey structure representing the foreign key constraint on table -** "t2". Calling this function with "t2" as the argument would return a -** NULL pointer (as there are no FK constraints for which t2 is the parent -** table). +** This function is called by the parser when it parses a command to create, +** release or rollback an SQL savepoint. */ -SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ - int nName = sqlite3Strlen30(pTab->zName); - return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); +SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + if( zName ){ + Vdbe *v = sqlite3GetVdbe(pParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + static const char * const az[] = { "BEGIN", "RELEASE", "ROLLBACK" }; + assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 ); +#endif + if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){ + sqlite3DbFree(pParse->db, zName); + return; + } + sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC); + } } /* -** The second argument is a Trigger structure allocated by the -** fkActionTrigger() routine. This function deletes the Trigger structure -** and all of its sub-components. -** -** The Trigger structure or any of its sub-components may be allocated from -** the lookaside buffer belonging to database handle dbMem. +** Make sure the TEMP database is open and available for use. Return +** the number of errors. Leave any error messages in the pParse structure. */ -static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ - if( p ){ - TriggerStep *pStep = p->step_list; - sqlite3ExprDelete(dbMem, pStep->pWhere); - sqlite3ExprListDelete(dbMem, pStep->pExprList); - sqlite3SelectDelete(dbMem, pStep->pSelect); - sqlite3ExprDelete(dbMem, p->pWhen); - sqlite3DbFree(dbMem, p); +SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt==0 && !pParse->explain ){ + int rc; + Btree *pBt; + static const int flags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TEMP_DB; + + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pBt, 0, flags); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "unable to open a temporary database " + "file for storing temporary tables"); + pParse->rc = rc; + return 1; + } + db->aDb[1].pBt = pBt; + assert( db->aDb[1].pSchema ); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + db->mallocFailed = 1; + return 1; + } } + return 0; } /* -** This function is called to generate code that runs when table pTab is -** being dropped from the database. The SrcList passed as the second argument -** to this function contains a single entry guaranteed to resolve to -** table pTab. +** Generate VDBE code that will verify the schema cookie and start +** a read-transaction for all named database files. ** -** Normally, no code is required. However, if either +** It is important that all schema cookies be verified and all +** read transactions be started before anything else happens in +** the VDBE program. But this routine can be called after much other +** code has been generated. So here is what we do: ** -** (a) The table is the parent table of a FK constraint, or -** (b) The table is the child table of a deferred FK constraint and it is -** determined at runtime that there are outstanding deferred FK -** constraint violations in the database, +** The first time this routine is called, we code an OP_Goto that +** will jump to a subroutine at the end of the program. Then we +** record every database that needs its schema verified in the +** pParse->cookieMask field. Later, after all other code has been +** generated, the subroutine that does the cookie verifications and +** starts the transactions will be coded and the OP_Goto P2 value +** will be made to point to that subroutine. The generation of the +** cookie verification subroutine code happens in sqlite3FinishCoding(). ** -** then the equivalent of "DELETE FROM " is executed before dropping -** the table from the database. Triggers are disabled while running this -** DELETE, but foreign key actions are not. +** If iDb<0 then code the OP_Goto only - don't set flag to verify the +** schema on any databases. This can be used to position the OP_Goto +** early in the code, before we know if any database tables will be used. */ -SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ - sqlite3 *db = pParse->db; - if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) && !pTab->pSelect ){ - int iSkip = 0; - Vdbe *v = sqlite3GetVdbe(pParse); +SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); - assert( v ); /* VDBE has already been allocated */ - if( sqlite3FkReferences(pTab)==0 ){ - /* Search for a deferred foreign key constraint for which this table - ** is the child table. If one cannot be found, return without - ** generating any VDBE code. If one can be found, then jump over - ** the entire DELETE if there are no outstanding deferred constraints - ** when this statement is run. */ - FKey *p; - for(p=pTab->pFKey; p; p=p->pNextFrom){ - if( p->isDeferred ) break; + if( pToplevel->cookieGoto==0 ){ + Vdbe *v = sqlite3GetVdbe(pToplevel); + if( v==0 ) return; /* This only happens if there was a prior error */ + pToplevel->cookieGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0)+1; + } + if( iDb>=0 ){ + sqlite3 *db = pToplevel->db; + yDbMask mask; + + assert( iDbnDb ); + assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbcookieMask & mask)==0 ){ + pToplevel->cookieMask |= mask; + pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; + if( !OMIT_TEMPDB && iDb==1 ){ + sqlite3OpenTempDatabase(pToplevel); } - if( !p ) return; - iSkip = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); } + } +} - pParse->disableTriggers = 1; - sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0); - pParse->disableTriggers = 0; - - /* If the DELETE has generated immediate foreign key constraint - ** violations, halt the VDBE and return an error at this point, before - ** any modifications to the schema are made. This is because statement - ** transactions are not able to rollback schema changes. */ - sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); - sqlite3HaltConstraint( - pParse, OE_Abort, "foreign key constraint failed", P4_STATIC - ); - - if( iSkip ){ - sqlite3VdbeResolveLabel(v, iSkip); +/* +** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each +** attached database. Otherwise, invoke it for the database named zDb only. +*/ +SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ + sqlite3 *db = pParse->db; + int i; + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ + sqlite3CodeVerifySchema(pParse, i); } } } /* -** This function is called when inserting, deleting or updating a row of -** table pTab to generate VDBE code to perform foreign key constraint -** processing for the operation. -** -** For a DELETE operation, parameter regOld is passed the index of the -** first register in an array of (pTab->nCol+1) registers containing the -** rowid of the row being deleted, followed by each of the column values -** of the row being deleted, from left to right. Parameter regNew is passed -** zero in this case. -** -** For an INSERT operation, regOld is passed zero and regNew is passed the -** first register of an array of (pTab->nCol+1) registers containing the new -** row data. +** Generate VDBE code that prepares for doing an operation that +** might change the database. ** -** For an UPDATE operation, this function is called twice. Once before -** the original record is deleted from the table using the calling convention -** described for DELETE. Then again after the original record is deleted -** but before the new record is inserted using the INSERT convention. +** This routine starts a new transaction if we are not already within +** a transaction. If we are already within a transaction, then a checkpoint +** is set if the setStatement parameter is true. A checkpoint should +** be set for operations that might fail (due to a constraint) part of +** the way through and which will need to undo some writes without having to +** rollback the whole transaction. For operations where all constraints +** can be checked before any changes are made to the database, it is never +** necessary to undo a write and the checkpoint should not be set. */ -SQLITE_PRIVATE void sqlite3FkCheck( - Parse *pParse, /* Parse context */ - Table *pTab, /* Row is being deleted from this table */ - int regOld, /* Previous row data is stored here */ - int regNew /* New row data is stored here */ -){ - sqlite3 *db = pParse->db; /* Database handle */ - Vdbe *v; /* VM to write code to */ - FKey *pFKey; /* Used to iterate through FKs */ - int iDb; /* Index of database containing pTab */ - const char *zDb; /* Name of database containing pTab */ - int isIgnoreErrors = pParse->disableTriggers; +SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3CodeVerifySchema(pParse, iDb); + pToplevel->writeMask |= ((yDbMask)1)<isMultiWrite |= setStatement; +} - /* Exactly one of regOld and regNew should be non-zero. */ - assert( (regOld==0)!=(regNew==0) ); - - /* If foreign-keys are disabled, this function is a no-op. */ - if( (db->flags&SQLITE_ForeignKeys)==0 ) return; +/* +** Indicate that the statement currently under construction might write +** more than one entry (example: deleting one row then inserting another, +** inserting multiple rows in a table, or inserting a row and index entries.) +** If an abort occurs after some of these writes have completed, then it will +** be necessary to undo the completed writes. +*/ +SQLITE_PRIVATE void sqlite3MultiWrite(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->isMultiWrite = 1; +} - v = sqlite3GetVdbe(pParse); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zName; +/* +** The code generator calls this routine if is discovers that it is +** possible to abort a statement prior to completion. In order to +** perform this abort without corrupting the database, we need to make +** sure that the statement is protected by a statement transaction. +** +** Technically, we only need to set the mayAbort flag if the +** isMultiWrite flag was previously set. There is a time dependency +** such that the abort must occur after the multiwrite. This makes +** some statements involving the REPLACE conflict resolution algorithm +** go a little faster. But taking advantage of this time dependency +** makes it more difficult to prove that the code is correct (in +** particular, it prevents us from writing an effective +** implementation of sqlite3AssertMayAbort()) and so we have chosen +** to take the safe route and skip the optimization. +*/ +SQLITE_PRIVATE void sqlite3MayAbort(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->mayAbort = 1; +} - /* Loop through all the foreign key constraints for which pTab is the - ** child table (the table that the foreign key definition is part of). */ - for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ - Table *pTo; /* Parent table of foreign key pFKey */ - Index *pIdx = 0; /* Index on key columns in pTo */ - int *aiFree = 0; - int *aiCol; - int iCol; - int i; - int isIgnore = 0; +/* +** Code an OP_Halt that causes the vdbe to return an SQLITE_CONSTRAINT +** error. The onError parameter determines which (if any) of the statement +** and/or current transaction is rolled back. +*/ +SQLITE_PRIVATE void sqlite3HaltConstraint(Parse *pParse, int onError, char *p4, int p4type){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( onError==OE_Abort ){ + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, p4, p4type); +} - /* Find the parent table of this foreign key. Also find a unique index - ** on the parent key columns in the parent table. If either of these - ** schema items cannot be located, set an error in pParse and return - ** early. */ - if( pParse->disableTriggers ){ - pTo = sqlite3FindTable(db, pFKey->zTo, zDb); - }else{ - pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); - } - if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ - if( !isIgnoreErrors || db->mallocFailed ) return; - continue; +/* +** Check to see if pIndex uses the collating sequence pColl. Return +** true if it does and false if it does not. +*/ +#ifndef SQLITE_OMIT_REINDEX +static int collationMatch(const char *zColl, Index *pIndex){ + int i; + assert( zColl!=0 ); + for(i=0; inColumn; i++){ + const char *z = pIndex->azColl[i]; + assert( z!=0 ); + if( 0==sqlite3StrICmp(z, zColl) ){ + return 1; } - assert( pFKey->nCol==1 || (aiFree && pIdx) ); + } + return 0; +} +#endif - if( aiFree ){ - aiCol = aiFree; - }else{ - iCol = pFKey->aCol[0].iFrom; - aiCol = &iCol; +/* +** Recompute all indices of pTab that use the collating sequence pColl. +** If pColl==0 then recompute all indices of pTab. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ + Index *pIndex; /* An index associated with pTab */ + + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( zColl==0 || collationMatch(zColl, pIndex) ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); } - for(i=0; inCol; i++){ - if( aiCol[i]==pTab->iPKey ){ - aiCol[i] = -1; - } -#ifndef SQLITE_OMIT_AUTHORIZATION - /* Request permission to read the parent key columns. If the - ** authorization callback returns SQLITE_IGNORE, behave as if any - ** values read from the parent table are NULL. */ - if( db->xAuth ){ - int rcauth; - char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; - rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); - isIgnore = (rcauth==SQLITE_IGNORE); - } + } +} #endif - } - /* Take a shared-cache advisory read-lock on the parent table. Allocate - ** a cursor to use to search the unique index on the parent key columns - ** in the parent table. */ - sqlite3TableLock(pParse, iDb, pTo->tnum, 0, pTo->zName); - pParse->nTab++; +/* +** Recompute all indices of all tables in all databases where the +** indices use the collating sequence pColl. If pColl==0 then recompute +** all indices everywhere. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexDatabases(Parse *pParse, char const *zColl){ + Db *pDb; /* A single database */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + HashElem *k; /* For looping over tables in pDb */ + Table *pTab; /* A table in the database */ - if( regOld!=0 ){ - /* A row is being removed from the child table. Search for the parent. - ** If the parent does not exist, removing the child row resolves an - ** outstanding foreign key constraint violation. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); - } - if( regNew!=0 ){ - /* A row is being added to the child table. If a parent row cannot - ** be found, adding the child row has violated the FK constraint. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); + assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + assert( pDb!=0 ); + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + reindexTable(pParse, pTab, zColl); } - - sqlite3DbFree(db, aiFree); } +} +#endif - /* Loop through all the foreign key constraints that refer to this table */ - for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ - Index *pIdx = 0; /* Foreign key index for pFKey */ - SrcList *pSrc; - int *aiCol = 0; +/* +** Generate code for the REINDEX command. +** +** REINDEX -- 1 +** REINDEX -- 2 +** REINDEX ?.? -- 3 +** REINDEX ?.? -- 4 +** +** Form 1 causes all indices in all attached databases to be rebuilt. +** Form 2 rebuilds all indices in all databases that use the named +** collating function. Forms 3 and 4 rebuild the named index or all +** indices associated with the named table. +*/ +#ifndef SQLITE_OMIT_REINDEX +SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ + CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ + char *z; /* Name of a table or index */ + const char *zDb; /* Name of the database */ + Table *pTab; /* A table in the database */ + Index *pIndex; /* An index associated with pTab */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + Token *pObjName; /* Name of the table or index to be reindexed */ - if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){ - assert( regOld==0 && regNew!=0 ); - /* Inserting a single row into a parent table cannot cause an immediate - ** foreign key violation. So do nothing in this case. */ - continue; - } + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ - if( !isIgnoreErrors || db->mallocFailed ) return; - continue; + if( pName1==0 ){ + reindexDatabases(pParse, 0); + return; + }else if( NEVER(pName2==0) || pName2->z==0 ){ + char *zColl; + assert( pName1->z ); + zColl = sqlite3NameFromToken(pParse->db, pName1); + if( !zColl ) return; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + if( pColl ){ + reindexDatabases(pParse, zColl); + sqlite3DbFree(db, zColl); + return; } - assert( aiCol || pFKey->nCol==1 ); + sqlite3DbFree(db, zColl); + } + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iDb].zName; + pTab = sqlite3FindTable(db, z, zDb); + if( pTab ){ + reindexTable(pParse, pTab, 0); + sqlite3DbFree(db, z); + return; + } + pIndex = sqlite3FindIndex(db, z, zDb); + sqlite3DbFree(db, z); + if( pIndex ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + return; + } + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); +} +#endif - /* Create a SrcList structure containing a single table (the table - ** the foreign key that refers to this table is attached to). This - ** is required for the sqlite3WhereXXX() interface. */ - pSrc = sqlite3SrcListAppend(db, 0, 0, 0); - if( pSrc ){ - struct SrcList_item *pItem = pSrc->a; - pItem->pTab = pFKey->pFrom; - pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nRef++; - pItem->iCursor = pParse->nTab++; - - if( regNew!=0 ){ - fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); - } - if( regOld!=0 ){ - /* If there is a RESTRICT action configured for the current operation - ** on the parent table of this FK, then throw an exception - ** immediately if the FK constraint is violated, even if this is a - ** deferred trigger. That's what RESTRICT means. To defer checking - ** the constraint, the FK should specify NO ACTION (represented - ** using OE_None). NO ACTION is the default. */ - fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); - } - pItem->zName = 0; - sqlite3SrcListDelete(db, pSrc); +/* +** Return a dynamicly allocated KeyInfo structure that can be used +** with OP_OpenRead or OP_OpenWrite to access database index pIdx. +** +** If successful, a pointer to the new structure is returned. In this case +** the caller is responsible for calling sqlite3DbFree(db, ) on the returned +** pointer. If an error occurs (out of memory or missing collation +** sequence), NULL is returned and the state of pParse updated to reflect +** the error. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nBytes = sizeof(KeyInfo) + (nCol-1)*sizeof(CollSeq*) + nCol; + sqlite3 *db = pParse->db; + KeyInfo *pKey = (KeyInfo *)sqlite3DbMallocZero(db, nBytes); + + if( pKey ){ + pKey->db = pParse->db; + pKey->aSortOrder = (u8 *)&(pKey->aColl[nCol]); + assert( &pKey->aSortOrder[nCol]==&(((u8 *)pKey)[nBytes]) ); + for(i=0; iazColl[i]; + assert( zColl ); + pKey->aColl[i] = sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } - sqlite3DbFree(db, aiCol); + pKey->nField = (u16)nCol; + } + + if( pParse->nErr ){ + sqlite3DbFree(db, pKey); + pKey = 0; } + return pKey; } -#define COLUMN_MASK(x) (((x)>31) ? 0xffffffff : ((u32)1<<(x))) +/************** End of build.c ***********************************************/ +/************** Begin file callback.c ****************************************/ +/* +** 2005 May 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains functions used to access the internal hash tables +** of user defined functions and collation sequences. +*/ + /* -** This function is called before generating code to update or delete a -** row contained in table pTab. +** Invoke the 'collation needed' callback to request a collation sequence +** in the encoding enc of name zName, length nName. */ -SQLITE_PRIVATE u32 sqlite3FkOldmask( - Parse *pParse, /* Parse context */ - Table *pTab /* Table being modified */ -){ - u32 mask = 0; - if( pParse->db->flags&SQLITE_ForeignKeys ){ - FKey *p; - int i; - for(p=pTab->pFKey; p; p=p->pNextFrom){ - for(i=0; inCol; i++) mask |= COLUMN_MASK(p->aCol[i].iFrom); +static void callCollNeeded(sqlite3 *db, int enc, const char *zName){ + assert( !db->xCollNeeded || !db->xCollNeeded16 ); + if( db->xCollNeeded ){ + char *zExternal = sqlite3DbStrDup(db, zName); + if( !zExternal ) return; + db->xCollNeeded(db->pCollNeededArg, db, enc, zExternal); + sqlite3DbFree(db, zExternal); + } +#ifndef SQLITE_OMIT_UTF16 + if( db->xCollNeeded16 ){ + char const *zExternal; + sqlite3_value *pTmp = sqlite3ValueNew(db); + sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF8, SQLITE_STATIC); + zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE); + if( zExternal ){ + db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); } - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - Index *pIdx = 0; - locateFkeyIndex(pParse, pTab, p, &pIdx, 0); - if( pIdx ){ - for(i=0; inColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); - } + sqlite3ValueFree(pTmp); + } +#endif +} + +/* +** This routine is called if the collation factory fails to deliver a +** collation function in the best encoding but there may be other versions +** of this collation function (for other text encodings) available. Use one +** of these instead if they exist. Avoid a UTF-8 <-> UTF-16 conversion if +** possible. +*/ +static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ + CollSeq *pColl2; + char *z = pColl->zName; + int i; + static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 }; + for(i=0; i<3; i++){ + pColl2 = sqlite3FindCollSeq(db, aEnc[i], z, 0); + if( pColl2->xCmp!=0 ){ + memcpy(pColl, pColl2, sizeof(CollSeq)); + pColl->xDel = 0; /* Do not copy the destructor */ + return SQLITE_OK; } } - return mask; + return SQLITE_ERROR; } /* -** This function is called before generating code to update or delete a -** row contained in table pTab. If the operation is a DELETE, then -** parameter aChange is passed a NULL value. For an UPDATE, aChange points -** to an array of size N, where N is the number of columns in table pTab. -** If the i'th column is not modified by the UPDATE, then the corresponding -** entry in the aChange[] array is set to -1. If the column is modified, -** the value is 0 or greater. Parameter chngRowid is set to true if the -** UPDATE statement modifies the rowid fields of the table. +** This function is responsible for invoking the collation factory callback +** or substituting a collation sequence of a different encoding when the +** requested collation sequence is not available in the desired encoding. +** +** If it is not NULL, then pColl must point to the database native encoding +** collation sequence with name zName, length nName. ** -** If any foreign key processing will be required, this function returns -** true. If there is no foreign key related processing, this function -** returns false. +** The return value is either the collation sequence to be used in database +** db for collation type name zName, length nName, or NULL, if no collation +** sequence can be found. +** +** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() */ -SQLITE_PRIVATE int sqlite3FkRequired( - Parse *pParse, /* Parse context */ - Table *pTab, /* Table being modified */ - int *aChange, /* Non-NULL for UPDATE operations */ - int chngRowid /* True for UPDATE that affects rowid */ +SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( + sqlite3* db, /* The database connection */ + u8 enc, /* The desired encoding for the collating sequence */ + CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ + const char *zName /* Collating sequence name */ ){ - if( pParse->db->flags&SQLITE_ForeignKeys ){ - if( !aChange ){ - /* A DELETE operation. Foreign key processing is required if the - ** table in question is either the child or parent table for any - ** foreign key constraint. */ - return (sqlite3FkReferences(pTab) || pTab->pFKey); - }else{ - /* This is an UPDATE. Foreign key processing is only required if the - ** operation modifies one or more child or parent key columns. */ - int i; - FKey *p; + CollSeq *p; - /* Check if any child key columns are being modified. */ - for(p=pTab->pFKey; p; p=p->pNextFrom){ - for(i=0; inCol; i++){ - int iChildKey = p->aCol[i].iFrom; - if( aChange[iChildKey]>=0 ) return 1; - if( iChildKey==pTab->iPKey && chngRowid ) return 1; - } - } + p = pColl; + if( !p ){ + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( !p || !p->xCmp ){ + /* No collation sequence of this type for this encoding is registered. + ** Call the collation factory to see if it can supply us with one. + */ + callCollNeeded(db, enc, zName); + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( p && !p->xCmp && synthCollSeq(db, p) ){ + p = 0; + } + assert( !p || p->xCmp ); + return p; +} - /* Check if any parent key columns are being modified. */ - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - for(i=0; inCol; i++){ - char *zKey = p->aCol[i].zCol; - int iKey; - for(iKey=0; iKeynCol; iKey++){ - Column *pCol = &pTab->aCol[iKey]; - if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ - if( aChange[iKey]>=0 ) return 1; - if( iKey==pTab->iPKey && chngRowid ) return 1; - } - } - } - } +/* +** This routine is called on a collation sequence before it is used to +** check that it is defined. An undefined collation sequence exists when +** a database is loaded that contains references to collation sequences +** that have not been defined by sqlite3_create_collation() etc. +** +** If required, this routine calls the 'collation needed' callback to +** request a definition of the collating sequence. If this doesn't work, +** an equivalent collating sequence that uses a text encoding different +** from the main database is substituted, if one is available. +*/ +SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ + if( pColl ){ + const char *zName = pColl->zName; + sqlite3 *db = pParse->db; + CollSeq *p = sqlite3GetCollSeq(db, ENC(db), pColl, zName); + if( !p ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + pParse->nErr++; + return SQLITE_ERROR; } + assert( p==pColl ); } - return 0; + return SQLITE_OK; } + + /* -** This function is called when an UPDATE or DELETE operation is being -** compiled on table pTab, which is the parent table of foreign-key pFKey. -** If the current operation is an UPDATE, then the pChanges parameter is -** passed a pointer to the list of columns being modified. If it is a -** DELETE, pChanges is passed a NULL pointer. -** -** It returns a pointer to a Trigger structure containing a trigger -** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. -** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is -** returned (these actions require no special handling by the triggers -** sub-system, code for them is created by fkScanChildren()). -** -** For example, if pFKey is the foreign key and pTab is table "p" in -** the following schema: -** -** CREATE TABLE p(pk PRIMARY KEY); -** CREATE TABLE c(ck REFERENCES p ON DELETE CASCADE); +** Locate and return an entry from the db.aCollSeq hash table. If the entry +** specified by zName and nName is not found and parameter 'create' is +** true, then create a new entry. Otherwise return NULL. ** -** then the returned trigger structure is equivalent to: +** Each pointer stored in the sqlite3.aCollSeq hash table contains an +** array of three CollSeq structures. The first is the collation sequence +** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. ** -** CREATE TRIGGER ... DELETE ON p BEGIN -** DELETE FROM c WHERE ck = old.pk; -** END; -** -** The returned pointer is cached as part of the foreign key object. It -** is eventually freed along with the rest of the foreign key object by -** sqlite3FkDelete(). +** Stored immediately after the three collation sequences is a copy of +** the collation sequence name. A pointer to this string is stored in +** each collation sequence structure. */ -static Trigger *fkActionTrigger( - Parse *pParse, /* Parse context */ - Table *pTab, /* Table being updated or deleted from */ - FKey *pFKey, /* Foreign key to get action for */ - ExprList *pChanges /* Change-list for UPDATE, NULL for DELETE */ +static CollSeq *findCollSeqEntry( + sqlite3 *db, /* Database connection */ + const char *zName, /* Name of the collating sequence */ + int create /* Create a new entry if true */ ){ - sqlite3 *db = pParse->db; /* Database handle */ - int action; /* One of OE_None, OE_Cascade etc. */ - Trigger *pTrigger; /* Trigger definition to return */ - int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ - - action = pFKey->aAction[iAction]; - pTrigger = pFKey->apTrigger[iAction]; - - if( action!=OE_None && !pTrigger ){ - u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ - char const *zFrom; /* Name of child table */ - int nFrom; /* Length in bytes of zFrom */ - Index *pIdx = 0; /* Parent key index for this FK */ - int *aiCol = 0; /* child table cols -> parent key cols */ - TriggerStep *pStep = 0; /* First (only) step of trigger program */ - Expr *pWhere = 0; /* WHERE clause of trigger step */ - ExprList *pList = 0; /* Changes list if ON UPDATE CASCADE */ - Select *pSelect = 0; /* If RESTRICT, "SELECT RAISE(...)" */ - int i; /* Iterator variable */ - Expr *pWhen = 0; /* WHEN clause for the trigger */ - - if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; - assert( aiCol || pFKey->nCol==1 ); - - for(i=0; inCol; i++){ - Token tOld = { "old", 3 }; /* Literal "old" token */ - Token tNew = { "new", 3 }; /* Literal "new" token */ - Token tFromCol; /* Name of column in child table */ - Token tToCol; /* Name of column in parent table */ - int iFromCol; /* Idx of column in child table */ - Expr *pEq; /* tFromCol = OLD.tToCol */ - - iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; - assert( iFromCol>=0 ); - tToCol.z = pIdx ? pTab->aCol[pIdx->aiColumn[i]].zName : "oid"; - tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; - - tToCol.n = sqlite3Strlen30(tToCol.z); - tFromCol.n = sqlite3Strlen30(tFromCol.z); + CollSeq *pColl; + int nName = sqlite3Strlen30(zName); + pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); - /* Create the expression "OLD.zToCol = zFromCol". It is important - ** that the "OLD.zToCol" term is on the LHS of the = operator, so - ** that the affinity and collation sequence associated with the - ** parent table are used for the comparison. */ - pEq = sqlite3PExpr(pParse, TK_EQ, - sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) - , 0), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tFromCol) - , 0); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + if( 0==pColl && create ){ + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); + if( pColl ){ + CollSeq *pDel = 0; + pColl[0].zName = (char*)&pColl[3]; + pColl[0].enc = SQLITE_UTF8; + pColl[1].zName = (char*)&pColl[3]; + pColl[1].enc = SQLITE_UTF16LE; + pColl[2].zName = (char*)&pColl[3]; + pColl[2].enc = SQLITE_UTF16BE; + memcpy(pColl[0].zName, zName, nName); + pColl[0].zName[nName] = 0; + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); - /* For ON UPDATE, construct the next term of the WHEN clause. - ** The final WHEN clause will be like this: - ** - ** WHEN NOT(old.col1 IS new.col1 AND ... AND old.colN IS new.colN) + /* If a malloc() failure occurred in sqlite3HashInsert(), it will + ** return the pColl pointer to be deleted (because it wasn't added + ** to the hash table). */ - if( pChanges ){ - pEq = sqlite3PExpr(pParse, TK_IS, - sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), - 0), - sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), - 0), - 0); - pWhen = sqlite3ExprAnd(db, pWhen, pEq); - } - - if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ - Expr *pNew; - if( action==OE_Cascade ){ - pNew = sqlite3PExpr(pParse, TK_DOT, - sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), - sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) - , 0); - }else if( action==OE_SetDflt ){ - Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; - if( pDflt ){ - pNew = sqlite3ExprDup(db, pDflt, 0); - }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); - } - }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); - } - pList = sqlite3ExprListAppend(pParse, pList, pNew); - sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); - } - } - sqlite3DbFree(db, aiCol); - - zFrom = pFKey->pFrom->zName; - nFrom = sqlite3Strlen30(zFrom); - - if( action==OE_Restrict ){ - Token tFrom; - Expr *pRaise; - - tFrom.z = zFrom; - tFrom.n = nFrom; - pRaise = sqlite3Expr(db, TK_RAISE, "foreign key constraint failed"); - if( pRaise ){ - pRaise->affinity = OE_Abort; + assert( pDel==0 || pDel==pColl ); + if( pDel!=0 ){ + db->mallocFailed = 1; + sqlite3DbFree(db, pDel); + pColl = 0; } - pSelect = sqlite3SelectNew(pParse, - sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(db, 0, &tFrom, 0), - pWhere, - 0, 0, 0, 0, 0, 0 - ); - pWhere = 0; } + } + return pColl; +} - /* Disable lookaside memory allocation */ - enableLookaside = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; +/* +** Parameter zName points to a UTF-8 encoded string nName bytes long. +** Return the CollSeq* pointer for the collation sequence named zName +** for the encoding 'enc' from the database 'db'. +** +** If the entry specified is not found and 'create' is true, then create a +** new entry. Otherwise return NULL. +** +** A separate function sqlite3LocateCollSeq() is a wrapper around +** this routine. sqlite3LocateCollSeq() invokes the collation factory +** if necessary and generates an error message if the collating sequence +** cannot be found. +** +** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( + sqlite3 *db, + u8 enc, + const char *zName, + int create +){ + CollSeq *pColl; + if( zName ){ + pColl = findCollSeqEntry(db, zName, create); + }else{ + pColl = db->pDfltColl; + } + assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); + if( pColl ) pColl += enc-1; + return pColl; +} - pTrigger = (Trigger *)sqlite3DbMallocZero(db, - sizeof(Trigger) + /* struct Trigger */ - sizeof(TriggerStep) + /* Single step in trigger program */ - nFrom + 1 /* Space for pStep->target.z */ - ); - if( pTrigger ){ - pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->target.z = (char *)&pStep[1]; - pStep->target.n = nFrom; - memcpy((char *)pStep->target.z, zFrom, nFrom); - - pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); - pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - if( pWhen ){ - pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0, 0); - pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); - } +/* During the search for the best function definition, this procedure +** is called to test how well the function passed as the first argument +** matches the request for a function with nArg arguments in a system +** that uses encoding enc. The value returned indicates how well the +** request is matched. A higher value indicates a better match. +** +** The returned value is always between 0 and 6, as follows: +** +** 0: Not a match, or if nArg<0 and the function is has no implementation. +** 1: A variable arguments function that prefers UTF-8 when a UTF-16 +** encoding is requested, or vice versa. +** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is +** requested, or vice versa. +** 3: A variable arguments function using the same text encoding. +** 4: A function with the exact number of arguments requested that +** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. +** 5: A function with the exact number of arguments requested that +** prefers UTF-16LE when UTF-16BE is requested, or vice versa. +** 6: An exact match. +** +*/ +static int matchQuality(FuncDef *p, int nArg, u8 enc){ + int match = 0; + if( p->nArg==-1 || p->nArg==nArg + || (nArg==-1 && (p->xFunc!=0 || p->xStep!=0)) + ){ + match = 1; + if( p->nArg==nArg || nArg==-1 ){ + match = 4; } - - /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bEnabled = enableLookaside; - - sqlite3ExprDelete(db, pWhere); - sqlite3ExprDelete(db, pWhen); - sqlite3ExprListDelete(db, pList); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed==1 ){ - fkTriggerDelete(db, pTrigger); - return 0; + if( enc==p->iPrefEnc ){ + match += 2; } - - switch( action ){ - case OE_Restrict: - pStep->op = TK_SELECT; - break; - case OE_Cascade: - if( !pChanges ){ - pStep->op = TK_DELETE; - break; - } - default: - pStep->op = TK_UPDATE; + else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || + (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ + match += 1; } - pStep->pTrig = pTrigger; - pTrigger->pSchema = pTab->pSchema; - pTrigger->pTabSchema = pTab->pSchema; - pFKey->apTrigger[iAction] = pTrigger; - pTrigger->op = (pChanges ? TK_UPDATE : TK_DELETE); } - - return pTrigger; + return match; } /* -** This function is called when deleting or updating a row to implement -** any required CASCADE, SET NULL or SET DEFAULT actions. +** Search a FuncDefHash for a function with the given name. Return +** a pointer to the matching FuncDef if found, or 0 if there is no match. */ -SQLITE_PRIVATE void sqlite3FkActions( - Parse *pParse, /* Parse context */ - Table *pTab, /* Table being updated or deleted from */ - ExprList *pChanges, /* Change-list for UPDATE, NULL for DELETE */ - int regOld /* Address of array containing old row */ +static FuncDef *functionSearch( + FuncDefHash *pHash, /* Hash table to search */ + int h, /* Hash of the name */ + const char *zFunc, /* Name of function */ + int nFunc /* Number of bytes in zFunc */ ){ - /* If foreign-key support is enabled, iterate through all FKs that - ** refer to table pTab. If there is an action associated with the FK - ** for this operation (either update or delete), invoke the associated - ** trigger sub-program. */ - if( pParse->db->flags&SQLITE_ForeignKeys ){ - FKey *pFKey; /* Iterator variable */ - for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ - Trigger *pAction = fkActionTrigger(pParse, pTab, pFKey, pChanges); - if( pAction ){ - sqlite3CodeRowTriggerDirect(pParse, pAction, pTab, regOld, OE_Abort, 0); - } + FuncDef *p; + for(p=pHash->a[h]; p; p=p->pHash){ + if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + return p; } } + return 0; } -#endif /* ifndef SQLITE_OMIT_TRIGGER */ +/* +** Insert a new FuncDef into a FuncDefHash hash table. +*/ +SQLITE_PRIVATE void sqlite3FuncDefInsert( + FuncDefHash *pHash, /* The hash table into which to insert */ + FuncDef *pDef /* The function definition to insert */ +){ + FuncDef *pOther; + int nName = sqlite3Strlen30(pDef->zName); + u8 c1 = (u8)pDef->zName[0]; + int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); + pOther = functionSearch(pHash, h, pDef->zName, nName); + if( pOther ){ + assert( pOther!=pDef && pOther->pNext!=pDef ); + pDef->pNext = pOther->pNext; + pOther->pNext = pDef; + }else{ + pDef->pNext = 0; + pDef->pHash = pHash->a[h]; + pHash->a[h] = pDef; + } +} + + /* -** Free all memory associated with foreign key definitions attached to -** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash -** hash table. +** Locate a user function given a name, a number of arguments and a flag +** indicating whether the function prefers UTF-16 over UTF-8. Return a +** pointer to the FuncDef structure that defines that function, or return +** NULL if the function does not exist. +** +** If the createFlag argument is true, then a new (blank) FuncDef +** structure is created and liked into the "db" structure if a +** no matching function previously existed. When createFlag is true +** and the nArg parameter is -1, then only a function that accepts +** any number of arguments will be returned. +** +** If createFlag is false and nArg is -1, then the first valid +** function found is returned. A function is valid if either xFunc +** or xStep is non-zero. +** +** If createFlag is false, then a function with the required name and +** number of arguments may be returned even if the eTextRep flag does not +** match that requested. */ -SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ - FKey *pFKey; /* Iterator variable */ - FKey *pNext; /* Copy of pFKey->pNextFrom */ +SQLITE_PRIVATE FuncDef *sqlite3FindFunction( + sqlite3 *db, /* An open database */ + const char *zName, /* Name of the function. Not null-terminated */ + int nName, /* Number of characters in the name */ + int nArg, /* Number of arguments. -1 means any number */ + u8 enc, /* Preferred text encoding */ + int createFlag /* Create new entry if true and does not otherwise exist */ +){ + FuncDef *p; /* Iterator variable */ + FuncDef *pBest = 0; /* Best match found so far */ + int bestScore = 0; /* Score of best match */ + int h; /* Hash value */ - for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ - /* Remove the FK from the fkeyHash hash table. */ - if( !db || db->pnBytesFreed==0 ){ - if( pFKey->pPrevTo ){ - pFKey->pPrevTo->pNextTo = pFKey->pNextTo; - }else{ - void *p = (void *)pFKey->pNextTo; - const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); - } - if( pFKey->pNextTo ){ - pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); + + /* First search for a match amongst the application-defined functions. + */ + p = functionSearch(&db->aFunc, h, zName, nName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + + /* If no match is found, search the built-in functions. + ** + ** If the SQLITE_PreferBuiltin flag is set, then search the built-in + ** functions even if a prior app-defined function was found. And give + ** priority to built-in functions. + ** + ** Except, if createFlag is true, that means that we are trying to + ** install a new function. Whatever FuncDef structure is returned it will + ** have fields overwritten with new information appropriate for the + ** new function. But the FuncDefs for built-in functions are read-only. + ** So we must not search for built-ins when creating a new function. + */ + if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + bestScore = 0; + p = functionSearch(pHash, h, zName, nName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; } + p = p->pNext; } + } - /* EV: R-30323-21917 Each foreign key constraint in SQLite is - ** classified as either immediate or deferred. - */ - assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + /* If the createFlag parameter is true and the search did not reveal an + ** exact match for the name, number of arguments and encoding, then add a + ** new entry to the hash table and return it. + */ + if( createFlag && (bestScore<6 || pBest->nArg!=nArg) && + (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ + pBest->zName = (char *)&pBest[1]; + pBest->nArg = (u16)nArg; + pBest->iPrefEnc = enc; + memcpy(pBest->zName, zName, nName); + pBest->zName[nName] = 0; + sqlite3FuncDefInsert(&db->aFunc, pBest); + } - /* Delete any triggers created to implement actions for this FK. */ -#ifndef SQLITE_OMIT_TRIGGER - fkTriggerDelete(db, pFKey->apTrigger[0]); - fkTriggerDelete(db, pFKey->apTrigger[1]); -#endif + if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ + return pBest; + } + return 0; +} - pNext = pFKey->pNextFrom; - sqlite3DbFree(db, pFKey); +/* +** Free all resources held by the schema structure. The void* argument points +** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the +** pointer itself, it just cleans up subsidiary resources (i.e. the contents +** of the schema hash tables). +** +** The Schema.cache_size variable is not cleared. +*/ +SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ + Hash temp1; + Hash temp2; + HashElem *pElem; + Schema *pSchema = (Schema *)p; + + temp1 = pSchema->tblHash; + temp2 = pSchema->trigHash; + sqlite3HashInit(&pSchema->trigHash); + sqlite3HashClear(&pSchema->idxHash); + for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ + sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + } + sqlite3HashClear(&temp2); + sqlite3HashInit(&pSchema->tblHash); + for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + sqlite3DeleteTable(0, pTab); + } + sqlite3HashClear(&temp1); + sqlite3HashClear(&pSchema->fkeyHash); + pSchema->pSeqTab = 0; + if( pSchema->flags & DB_SchemaLoaded ){ + pSchema->iGeneration++; + pSchema->flags &= ~DB_SchemaLoaded; } } -#endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ -/************** End of fkey.c ************************************************/ -/************** Begin file insert.c ******************************************/ +/* +** Find and return the schema associated with a BTree. Create +** a new one if necessary. +*/ +SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ + Schema * p; + if( pBt ){ + p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaClear); + }else{ + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); + } + if( !p ){ + db->mallocFailed = 1; + }else if ( 0==p->file_format ){ + sqlite3HashInit(&p->tblHash); + sqlite3HashInit(&p->idxHash); + sqlite3HashInit(&p->trigHash); + sqlite3HashInit(&p->fkeyHash); + p->enc = SQLITE_UTF8; + } + return p; +} + +/************** End of callback.c ********************************************/ +/************** Begin file delete.c ******************************************/ /* ** 2001 September 15 ** @@ -82278,3044 +81566,2253 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ ** ************************************************************************* ** This file contains C code routines that are called by the parser -** to handle INSERT statements in SQLite. +** in order to generate code for DELETE FROM statements. */ /* -** Generate code that will open a table for reading. +** While a SrcList can in general represent multiple tables and subqueries +** (as in the FROM clause of a SELECT statement) in this case it contains +** the name of a single table, as one might find in an INSERT, DELETE, +** or UPDATE statement. Look up that table in the symbol table and +** return a pointer. Set an error message and return NULL if the table +** name is not found or if any other error occurs. +** +** The following fields are initialized appropriate in pSrc: +** +** pSrc->a[0].pTab Pointer to the Table object +** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** */ -SQLITE_PRIVATE void sqlite3OpenTable( - Parse *p, /* Generate code into this VDBE */ - int iCur, /* The cursor number of the table */ - int iDb, /* The database index in sqlite3.aDb[] */ - Table *pTab, /* The table to be opened */ - int opcode /* OP_OpenRead or OP_OpenWrite */ -){ - Vdbe *v; - if( IsVirtual(pTab) ) return; - v = sqlite3GetVdbe(p); - assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); - sqlite3VdbeAddOp3(v, opcode, iCur, pTab->tnum, iDb); - sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(pTab->nCol), P4_INT32); - VdbeComment((v, "%s", pTab->zName)); -} - -/* -** Return a pointer to the column affinity string associated with index -** pIdx. A column affinity string has one character for each column in -** the table, according to the affinity of the column: -** -** Character Column affinity -** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL -** -** An extra 'b' is appended to the end of the string to cover the -** rowid that appears as the last column in every index. -** -** Memory for the buffer containing the column index affinity string -** is managed along with the rest of the Index structure. It will be -** released when sqlite3DeleteIndex() is called. -*/ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ - if( !pIdx->zColAff ){ - /* The first time a column affinity string for a particular index is - ** required, it is allocated and populated here. It is then stored as - ** a member of the Index structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqliteDeleteIndex() when the Index structure itself is cleaned - ** up. - */ - int n; - Table *pTab = pIdx->pTable; - sqlite3 *db = sqlite3VdbeDb(v); - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+2); - if( !pIdx->zColAff ){ - db->mallocFailed = 1; - return 0; - } - for(n=0; nnColumn; n++){ - pIdx->zColAff[n] = pTab->aCol[pIdx->aiColumn[n]].affinity; - } - pIdx->zColAff[n++] = SQLITE_AFF_NONE; - pIdx->zColAff[n] = 0; +SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ + struct SrcList_item *pItem = pSrc->a; + Table *pTab; + assert( pItem && pSrc->nSrc==1 ); + pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + sqlite3DeleteTable(pParse->db, pItem->pTab); + pItem->pTab = pTab; + if( pTab ){ + pTab->nRef++; } - - return pIdx->zColAff; + if( sqlite3IndexedByLookup(pParse, pItem) ){ + pTab = 0; + } + return pTab; } /* -** Set P4 of the most recently inserted opcode to a column affinity -** string for table pTab. A column affinity string has one character -** for each column indexed by the index, according to the affinity of the -** column: -** -** Character Column affinity -** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** Check to make sure the given table is writable. If it is not +** writable, generate an error message and return 1. If it is +** writable return 0; */ -SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ - /* The first time a column affinity string for a particular table - ** is required, it is allocated and populated here. It is then - ** stored as a member of the Table structure for subsequent use. +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ + /* A table is not writable under the following circumstances: ** - ** The column affinity string will eventually be deleted by - ** sqlite3DeleteTable() when the Table structure itself is cleaned up. + ** 1) It is a virtual table and no implementation of the xUpdate method + ** has been provided, or + ** 2) It is a system table (i.e. sqlite_master), this call is not + ** part of a nested parse and writable_schema pragma has not + ** been specified. + ** + ** In either case leave an error message in pParse and return non-zero. */ - if( !pTab->zColAff ){ - char *zColAff; - int i; - sqlite3 *db = sqlite3VdbeDb(v); - - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); - if( !zColAff ){ - db->mallocFailed = 1; - return; - } - - for(i=0; inCol; i++){ - zColAff[i] = pTab->aCol[i].affinity; - } - zColAff[pTab->nCol] = '\0'; - - pTab->zColAff = zColAff; + if( ( IsVirtual(pTab) + && sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ) + || ( (pTab->tabFlags & TF_Readonly)!=0 + && (pParse->db->flags & SQLITE_WriteSchema)==0 + && pParse->nested==0 ) + ){ + sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); + return 1; } - sqlite3VdbeChangeP4(v, -1, pTab->zColAff, 0); +#ifndef SQLITE_OMIT_VIEW + if( !viewOk && pTab->pSelect ){ + sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); + return 1; + } +#endif + return 0; } + +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) /* -** Return non-zero if the table pTab in database iDb or any of its indices -** have been opened at any point in the VDBE program beginning at location -** iStartAddr throught the end of the program. This is used to see if -** a statement of the form "INSERT INTO SELECT ..." can -** run without using temporary table for the results of the SELECT. +** Evaluate a view and store its result in an ephemeral table. The +** pWhere argument is an optional WHERE clause that restricts the +** set of rows in the view that are to be added to the ephemeral table. */ -static int readsTable(Parse *p, int iStartAddr, int iDb, Table *pTab){ - Vdbe *v = sqlite3GetVdbe(p); - int i; - int iEnd = sqlite3VdbeCurrentAddr(v); -#ifndef SQLITE_OMIT_VIRTUALTABLE - VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; -#endif +SQLITE_PRIVATE void sqlite3MaterializeView( + Parse *pParse, /* Parsing context */ + Table *pView, /* View definition */ + Expr *pWhere, /* Optional WHERE clause to be added */ + int iCur /* Cursor number for ephemerial table */ +){ + SelectDest dest; + Select *pDup; + sqlite3 *db = pParse->db; - for(i=iStartAddr; iopcode==OP_OpenRead && pOp->p3==iDb ){ - Index *pIndex; - int tnum = pOp->p2; - if( tnum==pTab->tnum ){ - return 1; - } - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( tnum==pIndex->tnum ){ - return 1; - } - } - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pOp->opcode==OP_VOpen && pOp->p4.pVtab==pVTab ){ - assert( pOp->p4.pVtab!=0 ); - assert( pOp->p4type==P4_VTAB ); - return 1; + pDup = sqlite3SelectDup(db, pView->pSelect, 0); + if( pWhere ){ + SrcList *pFrom; + + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zAlias = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].pSelect = pDup; + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); + }else{ + sqlite3SelectDelete(db, pDup); } -#endif + pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); } - return 0; + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); + sqlite3Select(pParse, pDup, &dest); + sqlite3SelectDelete(db, pDup); } +#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ -#ifndef SQLITE_OMIT_AUTOINCREMENT +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) /* -** Locate or create an AutoincInfo structure associated with table pTab -** which is in database iDb. Return the register number for the register -** that holds the maximum rowid. -** -** There is at most one AutoincInfo structure per table even if the -** same table is autoincremented multiple times due to inserts within -** triggers. A new AutoincInfo structure is created if this is the -** first use of table pTab. On 2nd and subsequent uses, the original -** AutoincInfo structure is used. -** -** Three memory locations are allocated: -** -** (1) Register to hold the name of the pTab table. -** (2) Register to hold the maximum ROWID of pTab. -** (3) Register to hold the rowid in sqlite_sequence of pTab +** Generate an expression tree to implement the WHERE, ORDER BY, +** and LIMIT/OFFSET portion of DELETE and UPDATE statements. ** -** The 2nd register is the one that is returned. That is all the -** insert routine needs to know about. +** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; +** \__________________________/ +** pLimitWhere (pInClause) */ -static int autoIncBegin( - Parse *pParse, /* Parsing context */ - int iDb, /* Index of the database holding pTab */ - Table *pTab /* The table we are writing to */ +SQLITE_PRIVATE Expr *sqlite3LimitWhere( + Parse *pParse, /* The parser context */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* The ORDER BY clause. May be null */ + Expr *pLimit, /* The LIMIT clause. May be null */ + Expr *pOffset, /* The OFFSET clause. May be null */ + char *zStmtType /* Either DELETE or UPDATE. For error messages. */ ){ - int memId = 0; /* Register holding maximum rowid */ - if( pTab->tabFlags & TF_Autoincrement ){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - AutoincInfo *pInfo; + Expr *pWhereRowid = NULL; /* WHERE rowid .. */ + Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ + Expr *pSelectRowid = NULL; /* SELECT rowid ... */ + ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ + Select *pSelect = NULL; /* Complete SELECT tree */ - pInfo = pToplevel->pAinc; - while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } - if( pInfo==0 ){ - pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); - if( pInfo==0 ) return 0; - pInfo->pNext = pToplevel->pAinc; - pToplevel->pAinc = pInfo; - pInfo->pTab = pTab; - pInfo->iDb = iDb; - pToplevel->nMem++; /* Register to hold name of table */ - pInfo->regCtr = ++pToplevel->nMem; /* Max rowid register */ - pToplevel->nMem++; /* Rowid in sqlite_sequence */ - } - memId = pInfo->regCtr; + /* Check that there isn't an ORDER BY without a LIMIT clause. + */ + if( pOrderBy && (pLimit == 0) ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); + pParse->parseError = 1; + goto limit_where_cleanup_2; } - return memId; -} -/* -** This routine generates code that will initialize all of the -** register used by the autoincrement tracker. -*/ -SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ - AutoincInfo *p; /* Information about an AUTOINCREMENT */ - sqlite3 *db = pParse->db; /* The database connection */ - Db *pDb; /* Database only autoinc table */ - int memId; /* Register holding max rowid */ - int addr; /* A VDBE address */ - Vdbe *v = pParse->pVdbe; /* VDBE under construction */ + /* We only need to generate a select expression if there + ** is a limit/offset term to enforce. + */ + if( pLimit == 0 ) { + /* if pLimit is null, pOffset will always be null as well. */ + assert( pOffset == 0 ); + return pWhere; + } - /* This routine is never called during trigger-generation. It is - ** only called from the top-level */ - assert( pParse->pTriggerTab==0 ); - assert( pParse==sqlite3ParseToplevel(pParse) ); + /* Generate a select expression tree to enforce the limit/offset + ** term for the DELETE or UPDATE statement. For example: + ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** becomes: + ** DELETE FROM table_a WHERE rowid IN ( + ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** ); + */ - assert( v ); /* We failed long ago if this is not so */ - for(p = pParse->pAinc; p; p = p->pNext){ - pDb = &db->aDb[p->iDb]; - memId = p->regCtr; - sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); - sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); - sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); - sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); - sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); - sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); - sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); - sqlite3VdbeAddOp0(v, OP_Close); + pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); + if( pSelectRowid == 0 ) goto limit_where_cleanup_2; + pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); + if( pEList == 0 ) goto limit_where_cleanup_2; + + /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree + ** and the SELECT subtree. */ + pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); + if( pSelectSrc == 0 ) { + sqlite3ExprListDelete(pParse->db, pEList); + goto limit_where_cleanup_2; } + + /* generate the SELECT expression tree. */ + pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0, + pOrderBy,0,pLimit,pOffset); + if( pSelect == 0 ) return 0; + + /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); + if( pWhereRowid == 0 ) goto limit_where_cleanup_1; + pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); + if( pInClause == 0 ) goto limit_where_cleanup_1; + + pInClause->x.pSelect = pSelect; + pInClause->flags |= EP_xIsSelect; + sqlite3ExprSetHeight(pParse, pInClause); + return pInClause; + + /* something went wrong. clean up anything allocated. */ +limit_where_cleanup_1: + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + +limit_where_cleanup_2: + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); + sqlite3ExprDelete(pParse->db, pOffset); + return 0; } +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ /* -** Update the maximum rowid for an autoincrement calculation. +** Generate code for a DELETE FROM statement. ** -** This routine should be called when the top of the stack holds a -** new rowid that is about to be inserted. If that new rowid is -** larger than the maximum rowid in the memId memory cell, then the -** memory cell is updated. The stack is unchanged. +** DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL; +** \________/ \________________/ +** pTabList pWhere */ -static void autoIncStep(Parse *pParse, int memId, int regRowid){ - if( memId>0 ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); +SQLITE_PRIVATE void sqlite3DeleteFrom( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table from which we should delete things */ + Expr *pWhere /* The WHERE clause. May be null */ +){ + Vdbe *v; /* The virtual database engine */ + Table *pTab; /* The table from which records will be deleted */ + const char *zDb; /* Name of database holding pTab */ + int end, addr = 0; /* A couple addresses of generated code */ + int i; /* Loop counter */ + WhereInfo *pWInfo; /* Information about the WHERE clause */ + Index *pIdx; /* For looping over indices of the table */ + int iCur; /* VDBE Cursor number for pTab */ + sqlite3 *db; /* Main database structure */ + AuthContext sContext; /* Authorization context */ + NameContext sNC; /* Name context to resolve expressions in */ + int iDb; /* Database number */ + int memCnt = -1; /* Memory cell used for change counting */ + int rcauth; /* Value returned by authorization callback */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to delete from a view */ + Trigger *pTrigger; /* List of table triggers, if required */ +#endif + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto delete_from_cleanup; } -} + assert( pTabList->nSrc==1 ); -/* -** This routine generates the code needed to write autoincrement -** maximum rowid values back into the sqlite_sequence register. -** Every statement that might do an INSERT into an autoincrement -** table (either directly or through triggers) needs to call this -** routine just before the "exit" code. -*/ -SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ - AutoincInfo *p; - Vdbe *v = pParse->pVdbe; - sqlite3 *db = pParse->db; + /* Locate the table which we want to delete. This table has to be + ** put in an SrcList structure because some of the subroutines we + ** will be calling are designed to work with multiple tables and expect + ** an SrcList* parameter instead of just a Table* parameter. + */ + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto delete_from_cleanup; - assert( v ); - for(p = pParse->pAinc; p; p = p->pNext){ - Db *pDb = &db->aDb[p->iDb]; - int j1, j2, j3, j4, j5; - int iRec; - int memId = p->regCtr; + /* Figure out if we have any triggers and if the table being + ** deleted from is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + isView = pTab->pSelect!=0; +#else +# define pTrigger 0 +# define isView 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif - iRec = sqlite3GetTempReg(pParse); - sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); - j2 = sqlite3VdbeAddOp0(v, OP_Rewind); - j3 = sqlite3VdbeAddOp3(v, OP_Column, 0, 0, iRec); - j4 = sqlite3VdbeAddOp3(v, OP_Eq, memId-1, 0, iRec); - sqlite3VdbeAddOp2(v, OP_Next, 0, j3); - sqlite3VdbeJumpHere(v, j2); - sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - j5 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, j4); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); - sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeJumpHere(v, j5); - sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); - sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3VdbeAddOp0(v, OP_Close); - sqlite3ReleaseTempReg(pParse, iRec); + /* If pTab is really a view, make sure it has been initialized. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto delete_from_cleanup; } -} -#else -/* -** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines -** above are all no-ops -*/ -# define autoIncBegin(A,B,C) (0) -# define autoIncStep(A,B,C) -#endif /* SQLITE_OMIT_AUTOINCREMENT */ + if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + goto delete_from_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + zDb = db->aDb[iDb].zName; + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); + if( rcauth==SQLITE_DENY ){ + goto delete_from_cleanup; + } + assert(!isView || pTrigger); -/* Forward declaration */ -static int xferOptimization( - Parse *pParse, /* Parser context */ - Table *pDest, /* The table we are inserting into */ - Select *pSelect, /* A SELECT statement to use as the data source */ - int onError, /* How to handle constraint errors */ - int iDbDest /* The database of pDest */ -); + /* Assign cursor number to the table and all its indices. + */ + assert( pTabList->nSrc==1 ); + iCur = pTabList->a[0].iCursor = pParse->nTab++; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + pParse->nTab++; + } -/* -** This routine is call to handle SQL of the following forms: -** -** insert into TABLE (IDLIST) values(EXPRLIST) -** insert into TABLE (IDLIST) select -** -** The IDLIST following the table name is always optional. If omitted, -** then a list of all columns for the table is substituted. The IDLIST -** appears in the pColumn parameter. pColumn is NULL if IDLIST is omitted. -** -** The pList parameter holds EXPRLIST in the first form of the INSERT -** statement above, and pSelect is NULL. For the second form, pList is -** NULL and pSelect is a pointer to the select statement used to generate -** data for the insert. -** -** The code generated follows one of four templates. For a simple -** select with data coming from a VALUES clause, the code executes -** once straight down through. Pseudo-code follows (we call this -** the "1st template"): -** -** open write cursor to
    and its indices -** puts VALUES clause expressions onto the stack -** write the resulting record into
    -** cleanup -** -** The three remaining templates assume the statement is of the form -** -** INSERT INTO
    SELECT ... -** -** If the SELECT clause is of the restricted form "SELECT * FROM " - -** in other words if the SELECT pulls all columns from a single table -** and there is no WHERE or LIMIT or GROUP BY or ORDER BY clauses, and -** if and are distinct tables but have identical -** schemas, including all the same indices, then a special optimization -** is invoked that copies raw records from over to . -** See the xferOptimization() function for the implementation of this -** template. This is the 2nd template. -** -** open a write cursor to
    -** open read cursor on -** transfer all records in over to
    -** close cursors -** foreach index on
    -** open a write cursor on the
    index -** open a read cursor on the corresponding index -** transfer all records from the read to the write cursors -** close cursors -** end foreach -** -** The 3rd template is for when the second template does not apply -** and the SELECT clause does not read from
    at any time. -** The generated code follows this template: -** -** EOF <- 0 -** X <- A -** goto B -** A: setup for the SELECT -** loop over the rows in the SELECT -** load values into registers R..R+n -** yield X -** end loop -** cleanup after the SELECT -** EOF <- 1 -** yield X -** goto A -** B: open write cursor to
    and its indices -** C: yield X -** if EOF goto D -** insert the select result into
    from R..R+n -** goto C -** D: cleanup -** -** The 4th template is used if the insert statement takes its -** values from a SELECT but the data is being inserted into a table -** that is also read as part of the SELECT. In the third form, -** we have to use a intermediate table to store the results of -** the select. The template is like this: -** -** EOF <- 0 -** X <- A -** goto B -** A: setup for the SELECT -** loop over the tables in the SELECT -** load value into register R..R+n -** yield X -** end loop -** cleanup after the SELECT -** EOF <- 1 -** yield X -** halt-error -** B: open temp table -** L: yield X -** if EOF goto M -** insert row from R..R+n into temp table -** goto L -** M: open write cursor to
    and its indices -** rewind temp table -** C: loop over rows of intermediate table -** transfer values form intermediate table into
    -** end loop -** D: cleanup -*/ -SQLITE_PRIVATE void sqlite3Insert( - Parse *pParse, /* Parser context */ - SrcList *pTabList, /* Name of table into which we are inserting */ - ExprList *pList, /* List of values to be inserted */ - Select *pSelect, /* A SELECT statement to use as the data source */ - IdList *pColumn, /* Column names corresponding to IDLIST. */ - int onError /* How to handle constraint errors */ -){ - sqlite3 *db; /* The main database structure */ - Table *pTab; /* The table to insert into. aka TABLE */ - char *zTab; /* Name of the table into which we are inserting */ - const char *zDb; /* Name of the database holding this table */ - int i, j, idx; /* Loop counters */ - Vdbe *v; /* Generate code into this virtual machine */ - Index *pIdx; /* For looping over indices of the table */ - int nColumn; /* Number of columns in the data */ - int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ - int baseCur = 0; /* VDBE Cursor number for pTab */ - int keyColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ - int endOfLoop; /* Label for the end of the insertion loop */ - int useTempTable = 0; /* Store SELECT results in intermediate table */ - int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ - int addrInsTop = 0; /* Jump to label "D" */ - int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ - int addrSelect = 0; /* Address of coroutine that implements the SELECT */ - SelectDest dest; /* Destination for SELECT on rhs of INSERT */ - int iDb; /* Index of database holding TABLE */ - Db *pDb; /* The database containing table being inserted into */ - int appendFlag = 0; /* True if the insert is likely to be an append */ - - /* Register allocations */ - int regFromSelect = 0;/* Base register for data coming from SELECT */ - int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ - int regRowCount = 0; /* Memory cell used for the row counter */ - int regIns; /* Block of regs holding rowid+data being inserted */ - int regRowid; /* registers holding insert rowid */ - int regData; /* register holding first column to insert */ - int regRecord; /* Holds the assemblied row record */ - int regEof = 0; /* Register recording end of SELECT data */ - int *aRegIdx = 0; /* One register allocated to each index */ - -#ifndef SQLITE_OMIT_TRIGGER - int isView; /* True if attempting to insert into a view */ - Trigger *pTrigger; /* List of triggers on pTab, if required */ - int tmask; /* Mask of trigger times */ -#endif - - db = pParse->db; - memset(&dest, 0, sizeof(dest)); - if( pParse->nErr || db->mallocFailed ){ - goto insert_cleanup; - } - - /* Locate the table into which we will be inserting new information. + /* Start the view context */ - assert( pTabList->nSrc==1 ); - zTab = pTabList->a[0].zName; - if( NEVER(zTab==0) ) goto insert_cleanup; - pTab = sqlite3SrcListLookup(pParse, pTabList); - if( pTab==0 ){ - goto insert_cleanup; - } - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( iDbnDb ); - pDb = &db->aDb[iDb]; - zDb = pDb->zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ - goto insert_cleanup; + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); } - /* Figure out if we have any triggers and if the table being - ** inserted into is a view - */ -#ifndef SQLITE_OMIT_TRIGGER - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); - isView = pTab->pSelect!=0; -#else -# define pTrigger 0 -# define tmask 0 -# define isView 0 -#endif -#ifdef SQLITE_OMIT_VIEW -# undef isView -# define isView 0 -#endif - assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); - - /* If pTab is really a view, make sure it has been initialized. - ** ViewGetColumnNames() is a no-op if pTab is not a view (or virtual - ** module table). + /* Begin generating code. */ - if( sqlite3ViewGetColumnNames(pParse, pTab) ){ - goto insert_cleanup; + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto delete_from_cleanup; } + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, 1, iDb); - /* Ensure that: - * (a) the table is not read-only, - * (b) that if it is a view then ON INSERT triggers exist + /* If we are trying to delete from a view, realize that view into + ** a ephemeral table. */ - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ - goto insert_cleanup; +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( isView ){ + sqlite3MaterializeView(pParse, pTab, pWhere, iCur); } +#endif - /* Allocate a VDBE + /* Resolve the column names in the WHERE clause. */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto insert_cleanup; - if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); - sqlite3BeginWriteOperation(pParse, pSelect || pTrigger, iDb); + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto delete_from_cleanup; + } -#ifndef SQLITE_OMIT_XFER_OPT - /* If the statement is of the form - ** - ** INSERT INTO SELECT * FROM ; - ** - ** Then special optimizations can be applied that make the transfer - ** very fast and which reduce fragmentation of indices. - ** - ** This is the 2nd template. + /* Initialize the counter of the number of rows deleted, if + ** we are counting rows. */ - if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ - assert( !pTrigger ); - assert( pList==0 ); - goto insert_end; + if( db->flags & SQLITE_CountRows ){ + memCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); } -#endif /* SQLITE_OMIT_XFER_OPT */ - /* If this is an AUTOINCREMENT table, look up the sequence number in the - ** sqlite_sequence table and store it in memory cell regAutoinc. +#ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + /* Special case: A DELETE without a WHERE clause deletes everything. + ** It is easier just to erase the whole table. Prior to version 3.6.5, + ** this optimization caused the row change count (the value returned by + ** API function sqlite3_count_changes) to be set incorrectly. */ + if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) + && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + ){ + assert( !isView ); + sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt, + pTab->zName, P4_STATIC); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->pSchema==pTab->pSchema ); + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); + } + }else +#endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ + /* The usual case: There is a WHERE clause so we have to scan through + ** the table and pick which records to delete. */ - regAutoinc = autoIncBegin(pParse, iDb, pTab); + { + int iRowSet = ++pParse->nMem; /* Register for rowset of rows to delete */ + int iRowid = ++pParse->nMem; /* Used for storing rowid values. */ + int regRowid; /* Actual register containing rowids */ - /* Figure out how many columns of data are supplied. If the data - ** is coming from a SELECT statement, then generate a co-routine that - ** produces a single row of the SELECT on each invocation. The - ** co-routine is the common header to the 3rd and 4th templates. - */ - if( pSelect ){ - /* Data is coming from a SELECT. Generate code to implement that SELECT - ** as a co-routine. The code is common to both the 3rd and 4th - ** templates: - ** - ** EOF <- 0 - ** X <- A - ** goto B - ** A: setup for the SELECT - ** loop over the tables in the SELECT - ** load value into register R..R+n - ** yield X - ** end loop - ** cleanup after the SELECT - ** EOF <- 1 - ** yield X - ** halt-error - ** - ** On each invocation of the co-routine, it puts a single row of the - ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. - ** (These output registers are allocated by sqlite3Select().) When - ** the SELECT completes, it sets the EOF flag stored in regEof. + /* Collect rowids of every row to be deleted. */ - int rc, j1; - - regEof = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ - VdbeComment((v, "SELECT eof flag")); - sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); - addrSelect = sqlite3VdbeCurrentAddr(v)+2; - sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iParm); - j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - VdbeComment((v, "Jump over SELECT coroutine")); - - /* Resolve the expressions in the SELECT statement and execute it. */ - rc = sqlite3Select(pParse, pSelect, &dest); - assert( pParse->nErr==0 || rc ); - if( rc || NEVER(pParse->nErr) || db->mallocFailed ){ - goto insert_cleanup; + sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); + if( pWInfo==0 ) goto delete_from_cleanup; + regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); + if( db->flags & SQLITE_CountRows ){ + sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); } - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); /* yield X */ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); - VdbeComment((v, "End of SELECT coroutine")); - sqlite3VdbeJumpHere(v, j1); /* label B: */ + sqlite3WhereEnd(pWInfo); - regFromSelect = dest.iMem; - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; - assert( dest.nMem==nColumn ); + /* Delete every item whose key was written to the list during the + ** database scan. We have to delete items after the scan is complete + ** because deleting an item can change the scan order. */ + end = sqlite3VdbeMakeLabel(v); - /* Set useTempTable to TRUE if the result of the SELECT statement - ** should be written into a temporary table (template 4). Set to - ** FALSE if each* row of the SELECT can be written directly into - ** the destination table (template 3). - ** - ** A temp table must be used if the table being updated is also one - ** of the tables being read by the SELECT statement. Also use a - ** temp table in the case of row triggers. - */ - if( pTrigger || readsTable(pParse, addrSelect, iDb, pTab) ){ - useTempTable = 1; + /* Unless this is a view, open cursors for the table we are + ** deleting from and all its indices. If this is a view, then the + ** only effect this statement has is to fire the INSTEAD OF + ** triggers. */ + if( !isView ){ + sqlite3OpenTableAndIndices(pParse, pTab, iCur, OP_OpenWrite); } - if( useTempTable ){ - /* Invoke the coroutine to extract information from the SELECT - ** and add it to a transient table srcTab. The code generated - ** here is from the 4th template: - ** - ** B: open temp table - ** L: yield X - ** if EOF goto M - ** insert row from R..R+n into temp table - ** goto L - ** M: ... - */ - int regRec; /* Register to hold packed record */ - int regTempRowid; /* Register to hold temp table ROWID */ - int addrTop; /* Label "L" */ - int addrIf; /* Address of jump to M */ + addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, end, iRowid); - srcTab = pParse->nTab++; - regRec = sqlite3GetTempReg(pParse); - regTempRowid = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); - addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); - addrIf = sqlite3VdbeAddOp1(v, OP_If, regEof); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); - sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); - sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - sqlite3VdbeJumpHere(v, addrIf); - sqlite3ReleaseTempReg(pParse, regRec); - sqlite3ReleaseTempReg(pParse, regTempRowid); - } - }else{ - /* This is the case if the data for the INSERT is coming from a VALUES - ** clause - */ - NameContext sNC; - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - srcTab = -1; - assert( useTempTable==0 ); - nColumn = pList ? pList->nExpr : 0; - for(i=0; ia[i].pExpr) ){ - goto insert_cleanup; - } + /* Delete the row */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); + sqlite3MayAbort(pParse); + }else +#endif + { + int count = (pParse->nested==0); /* True to count changes */ + sqlite3GenerateRowDelete(pParse, pTab, iCur, iRowid, count, pTrigger, OE_Default); } - } - /* Make sure the number of columns in the source data matches the number - ** of columns to be inserted into the table. - */ - if( IsVirtual(pTab) ){ - for(i=0; inCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); - } - } - if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ - sqlite3ErrorMsg(pParse, - "table %S has %d columns but %d values were supplied", - pTabList, 0, pTab->nCol-nHidden, nColumn); - goto insert_cleanup; - } - if( pColumn!=0 && nColumn!=pColumn->nId ){ - sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); - goto insert_cleanup; - } + /* End of the delete loop */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeResolveLabel(v, end); - /* If the INSERT statement included an IDLIST term, then make sure - ** all elements of the IDLIST really are columns of the table and - ** remember the column indices. - ** - ** If the table has an INTEGER PRIMARY KEY column and that column - ** is named in the IDLIST, then record in the keyColumn variable - ** the index into IDLIST of the primary key column. keyColumn is - ** the index of the primary key as it appears in IDLIST, not as - ** is appears in the original table. (The index of the primary - ** key in the original table is pTab->iPKey.) - */ - if( pColumn ){ - for(i=0; inId; i++){ - pColumn->a[i].idx = -1; - } - for(i=0; inId; i++){ - for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ - pColumn->a[i].idx = j; - if( j==pTab->iPKey ){ - keyColumn = i; - } - break; - } - } - if( j>=pTab->nCol ){ - if( sqlite3IsRowid(pColumn->a[i].zName) ){ - keyColumn = i; - }else{ - sqlite3ErrorMsg(pParse, "table %S has no column named %s", - pTabList, 0, pColumn->a[i].zName); - pParse->checkSchema = 1; - goto insert_cleanup; - } + /* Close the cursors open on the table and its indexes. */ + if( !isView && !IsVirtual(pTab) ){ + for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + sqlite3VdbeAddOp2(v, OP_Close, iCur + i, pIdx->tnum); } + sqlite3VdbeAddOp1(v, OP_Close, iCur); } } - /* If there is no IDLIST term but the table has an integer primary - ** key, the set the keyColumn variable to the primary key column index - ** in the original table definition. + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. */ - if( pColumn==0 && nColumn>0 ){ - keyColumn = pTab->iPKey; + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); } - - /* Initialize the count of rows to be inserted + + /* Return the number of rows that were deleted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. */ - if( db->flags & SQLITE_CountRows ){ - regRowCount = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); } - /* If this is not a view, open the table and and all indices */ - if( !isView ){ - int nIdx; - - baseCur = pParse->nTab; - nIdx = sqlite3OpenTableAndIndices(pParse, pTab, baseCur, OP_OpenWrite); - aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); - if( aRegIdx==0 ){ - goto insert_cleanup; - } - for(i=0; inMem; - } - } +delete_from_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprDelete(db, pWhere); + return; +} +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** thely may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif - /* This is the top of the main insertion loop */ - if( useTempTable ){ - /* This block codes the top of loop only. The complete loop is the - ** following pseudocode (template 4): - ** - ** rewind temp table - ** C: loop over rows of intermediate table - ** transfer values form intermediate table into
    - ** end loop - ** D: ... - */ - addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); - addrCont = sqlite3VdbeCurrentAddr(v); - }else if( pSelect ){ - /* This block codes the top of loop only. The complete loop is the - ** following pseudocode (template 3): - ** - ** C: yield X - ** if EOF goto D - ** insert the select result into
    from R..R+n - ** goto C - ** D: ... - */ - addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); - addrInsTop = sqlite3VdbeAddOp1(v, OP_If, regEof); - } - - /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assemblied row record. - */ - regRecord = ++pParse->nMem; - regRowid = regIns = pParse->nMem+1; - pParse->nMem += pTab->nCol + 1; - if( IsVirtual(pTab) ){ - regRowid++; - pParse->nMem++; - } - regData = regRowid+1; +/* +** This routine generates VDBE code that causes a single row of a +** single table to be deleted. +** +** The VDBE must be in a particular state when this routine is called. +** These are the requirements: +** +** 1. A read/write cursor pointing to pTab, the table containing the row +** to be deleted, must be opened as cursor number $iCur. +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number base+i for the i-th index. +** +** 3. The record number of the row to be deleted must be stored in +** memory cell iRowid. +** +** This routine generates code to remove both the table record and all +** index entries that point to that record. +*/ +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table containing the row to be deleted */ + int iCur, /* Cursor number for the table */ + int iRowid, /* Memory cell that contains the rowid to delete */ + int count, /* If non-zero, increment the row change counter */ + Trigger *pTrigger, /* List of triggers to (potentially) fire */ + int onconf /* Default ON CONFLICT policy for triggers */ +){ + Vdbe *v = pParse->pVdbe; /* Vdbe */ + int iOld = 0; /* First register in OLD.* array */ + int iLabel; /* Label resolved to end of generated code */ - /* Run the BEFORE and INSTEAD OF triggers, if there are any - */ - endOfLoop = sqlite3VdbeMakeLabel(v); - if( tmask & TRIGGER_BEFORE ){ - int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); + /* Vdbe is guaranteed to have been allocated by this stage. */ + assert( v ); - /* build the NEW.* reference row. Note that if there is an INTEGER - ** PRIMARY KEY into which a NULL is being inserted, that NULL will be - ** translated into a unique ID for the row. But on a BEFORE trigger, - ** we do not know what the unique ID will be (because the insert has - ** not happened yet) so we substitute a rowid of -1 - */ - if( keyColumn<0 ){ - sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - }else{ - int j1; - if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regCols); - }else{ - assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regCols); - } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); - sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); - } + /* Seek cursor iCur to the row to delete. If this row no longer exists + ** (this can happen if a trigger program has already deleted it), do + ** not attempt to delete it or fire any DELETE triggers. */ + iLabel = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, iLabel, iRowid); + + /* If there are any triggers to fire, allocate a range of registers to + ** use for the old.* references in the triggers. */ + if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ + u32 mask; /* Mask of OLD.* columns in use */ + int iCol; /* Iterator used while populating OLD.* */ - /* Cannot have triggers on a virtual table. If it were possible, - ** this block would have to account for hidden column. - */ - assert( !IsVirtual(pTab) ); + /* TODO: Could use temporary registers here. Also could attempt to + ** avoid copying the contents of the rowid register. */ + mask = sqlite3TriggerColmask( + pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf + ); + mask |= sqlite3FkOldmask(pParse, pTab); + iOld = pParse->nMem+1; + pParse->nMem += (1 + pTab->nCol); - /* Create the new column data - */ - for(i=0; inCol; i++){ - if( pColumn==0 ){ - j = i; - }else{ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); - }else{ - assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); + /* Populate the OLD.* pseudo-table register array. These values will be + ** used by any BEFORE and AFTER triggers that exist. */ + sqlite3VdbeAddOp2(v, OP_Copy, iRowid, iOld); + for(iCol=0; iColnCol; iCol++){ + if( mask==0xffffffff || mask&(1<nCol); - sqlite3TableAffinityStr(v, pTab); - } + /* Invoke BEFORE DELETE trigger programs. */ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel + ); - /* Fire BEFORE or INSTEAD OF triggers */ - sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, - pTab, regCols-pTab->nCol-1, onError, endOfLoop); + /* Seek the cursor to the row to be deleted again. It may be that + ** the BEFORE triggers coded above have already removed the row + ** being deleted. Do not attempt to delete the row a second time, and + ** do not fire AFTER triggers. */ + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, iLabel, iRowid); - sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); + /* Do FK processing. This call checks that any FK constraints that + ** refer to this table (i.e. constraints attached to other tables) + ** are not violated by deleting this row. */ + sqlite3FkCheck(pParse, pTab, iOld, 0); } - /* Push the record number for the new entry onto the stack. The - ** record number is a randomly generate integer created by NewRowid - ** except when the table has an INTEGER PRIMARY KEY column, in which - ** case the record number is the same as that column. - */ - if( !isView ){ - if( IsVirtual(pTab) ){ - /* The row that the VUpdate opcode will delete: none */ - sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); - } - if( keyColumn>=0 ){ - if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regRowid); - }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+keyColumn, regRowid); - }else{ - VdbeOp *pOp; - sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regRowid); - pOp = sqlite3VdbeGetOp(v, -1); - if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ - appendFlag = 1; - pOp->opcode = OP_NewRowid; - pOp->p1 = baseCur; - pOp->p2 = regRowid; - pOp->p3 = regAutoinc; - } - } - /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid - ** to generate a unique primary key value. - */ - if( !appendFlag ){ - int j1; - if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); - sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, j1); - }else{ - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); - } - sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); - } - }else if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); - }else{ - sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); - appendFlag = 1; + /* Delete the index and table entries. Skip this step if pTab is really + ** a view (in which case the only effect of the DELETE statement is to + ** fire the INSTEAD OF triggers). */ + if( pTab->pSelect==0 ){ + sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, 0); + sqlite3VdbeAddOp2(v, OP_Delete, iCur, (count?OPFLAG_NCHANGE:0)); + if( count ){ + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } - autoIncStep(pParse, regAutoinc, regRowid); + } - /* Push onto the stack, data for all columns of the new entry, beginning - ** with the first column. - */ - nHidden = 0; - for(i=0; inCol; i++){ - int iRegStore = regRowid+1+i; - if( i==pTab->iPKey ){ - /* The value of the INTEGER PRIMARY KEY column is always a NULL. - ** Whenever this column is read, the record number will be substituted - ** in its place. So will fill this column with a NULL to avoid - ** taking up data space with information that will never be used. */ - sqlite3VdbeAddOp2(v, OP_Null, 0, iRegStore); - continue; - } - if( pColumn==0 ){ - if( IsHiddenColumn(&pTab->aCol[i]) ){ - assert( IsVirtual(pTab) ); - j = -1; - nHidden++; - }else{ - j = i - nHidden; - } - }else{ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, iRegStore); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); - }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); - }else{ - sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); - } - } + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just deleted. */ + sqlite3FkActions(pParse, pTab, 0, iOld); - /* Generate code to check constraints and generate index keys and - ** do the insertion. - */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); - sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); - sqlite3MayAbort(pParse); - }else -#endif - { - int isReplace; /* Set to true if constraints may cause a replace */ - sqlite3GenerateConstraintChecks(pParse, pTab, baseCur, regIns, aRegIdx, - keyColumn>=0, 0, onError, endOfLoop, &isReplace - ); - sqlite3FkCheck(pParse, pTab, 0, regIns); - sqlite3CompleteInsertion( - pParse, pTab, baseCur, regIns, aRegIdx, 0, appendFlag, isReplace==0 - ); - } - } + /* Invoke AFTER DELETE trigger programs. */ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); - /* Update the count of rows that are inserted - */ - if( (db->flags & SQLITE_CountRows)!=0 ){ - sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); - } + /* Jump here if the row had already been deleted before any BEFORE + ** trigger programs were invoked. Or if a trigger program throws a + ** RAISE(IGNORE) exception. */ + sqlite3VdbeResolveLabel(v, iLabel); +} - if( pTrigger ){ - /* Code AFTER triggers */ - sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, - pTab, regData-2-pTab->nCol, onError, endOfLoop); - } +/* +** This routine generates VDBE code that causes the deletion of all +** index entries associated with a single row of a single table. +** +** The VDBE must be in a particular state when this routine is called. +** These are the requirements: +** +** 1. A read/write cursor pointing to pTab, the table containing the row +** to be deleted, must be opened as cursor number "iCur". +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iCur+i for the i-th index. +** +** 3. The "iCur" cursor must be pointing to the row that is to be +** deleted. +*/ +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Table containing the row to be deleted */ + int iCur, /* Cursor number for the table */ + int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ +){ + int i; + Index *pIdx; + int r1; - /* The bottom of the main insertion loop, if the data source - ** is a SELECT statement. - */ - sqlite3VdbeResolveLabel(v, endOfLoop); - if( useTempTable ){ - sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); - sqlite3VdbeJumpHere(v, addrInsTop); - sqlite3VdbeAddOp1(v, OP_Close, srcTab); - }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); - sqlite3VdbeJumpHere(v, addrInsTop); + for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + if( aRegIdx!=0 && aRegIdx[i-1]==0 ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iCur, 0, 0); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_IdxDelete, iCur+i, r1,pIdx->nColumn+1); } +} - if( !IsVirtual(pTab) && !isView ){ - /* Close all tables opened */ - sqlite3VdbeAddOp1(v, OP_Close, baseCur); - for(idx=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, idx++){ - sqlite3VdbeAddOp1(v, OP_Close, idx+baseCur); - } - } +/* +** Generate code that will assemble an index key and put it in register +** regOut. The key with be for index pIdx which is an index on pTab. +** iCur is the index of a cursor open on the pTab table and pointing to +** the entry that needs indexing. +** +** Return a register number which is the first in a block of +** registers that holds the elements of the index key. The +** block of registers has already been deallocated by the time +** this routine returns. +*/ +SQLITE_PRIVATE int sqlite3GenerateIndexKey( + Parse *pParse, /* Parsing context */ + Index *pIdx, /* The index for which to generate a key */ + int iCur, /* Cursor number for the pIdx->pTable table */ + int regOut, /* Write the new index key to this register */ + int doMakeRec /* Run the OP_MakeRecord instruction if true */ +){ + Vdbe *v = pParse->pVdbe; + int j; + Table *pTab = pIdx->pTable; + int regBase; + int nCol; -insert_end: - /* Update the sqlite_sequence table by storing the content of the - ** maximum rowid counter values recorded while inserting into - ** autoincrement tables. - */ - if( pParse->nested==0 && pParse->pTriggerTab==0 ){ - sqlite3AutoincrementEnd(pParse); + nCol = pIdx->nColumn; + regBase = sqlite3GetTempRange(pParse, nCol+1); + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regBase+nCol); + for(j=0; jaiColumn[j]; + if( idx==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regBase+nCol, regBase+j); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iCur, idx, regBase+j); + sqlite3ColumnDefault(v, pTab, idx, -1); + } } - - /* - ** Return the number of rows inserted. If this routine is - ** generating code because of a call to sqlite3NestedParse(), do not - ** invoke the callback function. - */ - if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); + if( doMakeRec ){ + const char *zAff; + if( pTab->pSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + zAff = 0; + }else{ + zAff = sqlite3IndexAffinityStr(v, pIdx); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); + sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); } - -insert_cleanup: - sqlite3SrcListDelete(db, pTabList); - sqlite3ExprListDelete(db, pList); - sqlite3SelectDelete(db, pSelect); - sqlite3IdListDelete(db, pColumn); - sqlite3DbFree(db, aRegIdx); + sqlite3ReleaseTempRange(pParse, regBase, nCol+1); + return regBase; } -/* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file -** (or in another file, if this file becomes part of the amalgamation). */ -#ifdef isView - #undef isView -#endif -#ifdef pTrigger - #undef pTrigger -#endif -#ifdef tmask - #undef tmask -#endif - - +/************** End of delete.c **********************************************/ +/************** Begin file func.c ********************************************/ /* -** Generate code to do constraint checks prior to an INSERT or an UPDATE. -** -** The input is a range of consecutive registers as follows: -** -** 1. The rowid of the row after the update. -** -** 2. The data in the first column of the entry after the update. -** -** i. Data from middle columns... -** -** N. The data in the last column of the entry after the update. -** -** The regRowid parameter is the index of the register containing (1). -** -** If isUpdate is true and rowidChng is non-zero, then rowidChng contains -** the address of a register containing the rowid before the update takes -** place. isUpdate is true for UPDATEs and false for INSERTs. If isUpdate -** is false, indicating an INSERT statement, then a non-zero rowidChng -** indicates that the rowid was explicitly specified as part of the -** INSERT statement. If rowidChng is false, it means that the rowid is -** computed automatically in an insert or that the rowid value is not -** modified by an update. -** -** The code generated by this routine store new index entries into -** registers identified by aRegIdx[]. No index entry is created for -** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is -** the same as the order of indices on the linked list of indices -** attached to the table. -** -** This routine also generates code to check constraints. NOT NULL, -** CHECK, and UNIQUE constraints are all checked. If a constraint fails, -** then the appropriate action is performed. There are five possible -** actions: ROLLBACK, ABORT, FAIL, REPLACE, and IGNORE. -** -** Constraint type Action What Happens -** --------------- ---------- ---------------------------------------- -** any ROLLBACK The current transaction is rolled back and -** sqlite3_exec() returns immediately with a -** return code of SQLITE_CONSTRAINT. -** -** any ABORT Back out changes from the current command -** only (do not do a complete rollback) then -** cause sqlite3_exec() to return immediately -** with SQLITE_CONSTRAINT. -** -** any FAIL Sqlite_exec() returns immediately with a -** return code of SQLITE_CONSTRAINT. The -** transaction is not rolled back and any -** prior changes are retained. -** -** any IGNORE The record number and data is popped from -** the stack and there is an immediate jump -** to label ignoreDest. -** -** NOT NULL REPLACE The NULL value is replace by the default -** value for that column. If the default value -** is NULL, the action is the same as ABORT. +** 2002 February 23 ** -** UNIQUE REPLACE The other row that conflicts with the row -** being inserted is removed. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** CHECK REPLACE Illegal. The results in an exception. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** Which action to take is determined by the overrideError parameter. -** Or if overrideError==OE_Default, then the pParse->onError parameter -** is used. Or if pParse->onError==OE_Default then the onError value -** for the constraint is used. +************************************************************************* +** This file contains the C functions that implement various SQL +** functions of SQLite. ** -** The calling routine must open a read/write cursor for pTab with -** cursor number "baseCur". All indices of pTab must also have open -** read/write cursors with cursor number baseCur+i for the i-th cursor. -** Except, if there is no possibility of a REPLACE action then -** cursors do not need to be open for indices where aRegIdx[i]==0. +** There is only one exported symbol in this file - the function +** sqliteRegisterBuildinFunctions() found at the bottom of the file. +** All other code has file scope. */ -SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( - Parse *pParse, /* The parser context */ - Table *pTab, /* the table into which we are inserting */ - int baseCur, /* Index of a read/write cursor pointing at pTab */ - int regRowid, /* Index of the range of input registers */ - int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int rowidChng, /* True if the rowid might collide with existing entry */ - int isUpdate, /* True for UPDATE, False for INSERT */ - int overrideError, /* Override onError to this if not OE_Default */ - int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ - int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ -){ - int i; /* loop counter */ - Vdbe *v; /* VDBE under constrution */ - int nCol; /* Number of columns */ - int onError; /* Conflict resolution strategy */ - int j1; /* Addresss of jump instruction */ - int j2 = 0, j3; /* Addresses of jump instructions */ - int regData; /* Register containing first data column */ - int iCur; /* Table cursor number */ - Index *pIdx; /* Pointer to one of the indices */ - int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ - int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; - - v = sqlite3GetVdbe(pParse); - assert( v!=0 ); - assert( pTab->pSelect==0 ); /* This table is not a VIEW */ - nCol = pTab->nCol; - regData = regRowid + 1; - - /* Test all NOT NULL constraints. - */ - for(i=0; iiPKey ){ - continue; - } - onError = pTab->aCol[i].notNull; - if( onError==OE_None ) continue; - if( overrideError!=OE_Default ){ - onError = overrideError; - }else if( onError==OE_Default ){ - onError = OE_Abort; - } - if( onError==OE_Replace && pTab->aCol[i].pDflt==0 ){ - onError = OE_Abort; - } - assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail - || onError==OE_Ignore || onError==OE_Replace ); - switch( onError ){ - case OE_Abort: - sqlite3MayAbort(pParse); - case OE_Rollback: - case OE_Fail: { - char *zMsg; - j1 = sqlite3VdbeAddOp3(v, OP_HaltIfNull, - SQLITE_CONSTRAINT, onError, regData+i); - zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", - pTab->zName, pTab->aCol[i].zName); - sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); - break; - } - case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_IsNull, regData+i, ignoreDest); - break; - } - default: { - assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regData+i); - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regData+i); - sqlite3VdbeJumpHere(v, j1); - break; - } - } - } - - /* Test all CHECK constraints - */ -#ifndef SQLITE_OMIT_CHECK - if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ - int allOk = sqlite3VdbeMakeLabel(v); - pParse->ckBase = regData; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); - onError = overrideError!=OE_Default ? overrideError : OE_Abort; - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - sqlite3HaltConstraint(pParse, onError, 0, 0); - } - sqlite3VdbeResolveLabel(v, allOk); - } -#endif /* !defined(SQLITE_OMIT_CHECK) */ - - /* If we have an INTEGER PRIMARY KEY, make sure the primary key - ** of the new record does not previously exist. Except, if this - ** is an UPDATE and the primary key is not changing, that is OK. - */ - if( rowidChng ){ - onError = pTab->keyConf; - if( overrideError!=OE_Default ){ - onError = overrideError; - }else if( onError==OE_Default ){ - onError = OE_Abort; - } - - if( isUpdate ){ - j2 = sqlite3VdbeAddOp3(v, OP_Eq, regRowid, 0, rowidChng); - } - j3 = sqlite3VdbeAddOp3(v, OP_NotExists, baseCur, 0, regRowid); - switch( onError ){ - default: { - onError = OE_Abort; - /* Fall thru into the next case */ - } - case OE_Rollback: - case OE_Abort: - case OE_Fail: { - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); - break; - } - case OE_Replace: { - /* If there are DELETE triggers on this table and the - ** recursive-triggers flag is set, call GenerateRowDelete() to - ** remove the conflicting row from the the table. This will fire - ** the triggers and remove both the table and index b-tree entries. - ** - ** Otherwise, if there are no triggers or the recursive-triggers - ** flag is not set, but the table has one or more indexes, call - ** GenerateRowIndexDelete(). This removes the index b-tree entries - ** only. The table b-tree entry will be replaced by the new entry - ** when it is inserted. - ** - ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, - ** also invoke MultiWrite() to indicate that this VDBE may require - ** statement rollback (if the statement is aborted after the delete - ** takes place). Earlier versions called sqlite3MultiWrite() regardless, - ** but being more selective here allows statements like: - ** - ** REPLACE INTO t(rowid) VALUES($newrowid) - ** - ** to run without a statement journal if there are no indexes on the - ** table. - */ - Trigger *pTrigger = 0; - if( pParse->db->flags&SQLITE_RecTriggers ){ - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - } - if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowDelete( - pParse, pTab, baseCur, regRowid, 0, pTrigger, OE_Replace - ); - }else if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, baseCur, 0); - } - seenReplace = 1; - break; - } - case OE_Ignore: { - assert( seenReplace==0 ); - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - break; - } - } - sqlite3VdbeJumpHere(v, j3); - if( isUpdate ){ - sqlite3VdbeJumpHere(v, j2); - } - } - - /* Test all UNIQUE constraints by creating entries for each UNIQUE - ** index and making sure that duplicate entries do not already exist. - ** Add the new records to the indices as we go. - */ - for(iCur=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, iCur++){ - int regIdx; - int regR; - - if( aRegIdx[iCur]==0 ) continue; /* Skip unused indices */ - - /* Create a key for accessing the index entry */ - regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn+1); - for(i=0; inColumn; i++){ - int idx = pIdx->aiColumn[i]; - if( idx==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, regData+idx, regIdx+i); - } - } - sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn+1, aRegIdx[iCur]); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0); - sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn+1); - - /* Find out what action to take in case there is an indexing conflict */ - onError = pIdx->onError; - if( onError==OE_None ){ - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); - continue; /* pIdx is not a UNIQUE index */ - } - if( overrideError!=OE_Default ){ - onError = overrideError; - }else if( onError==OE_Default ){ - onError = OE_Abort; - } - if( seenReplace ){ - if( onError==OE_Ignore ) onError = OE_Replace; - else if( onError==OE_Fail ) onError = OE_Abort; - } - - /* Check to see if the new index entry will be unique */ - regR = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_SCopy, regOldRowid, regR); - j3 = sqlite3VdbeAddOp4(v, OP_IsUnique, baseCur+iCur+1, 0, - regR, SQLITE_INT_TO_PTR(regIdx), - P4_INT32); - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); - /* Generate code that executes if the new index entry is not unique */ - assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail - || onError==OE_Ignore || onError==OE_Replace ); - switch( onError ){ - case OE_Rollback: - case OE_Abort: - case OE_Fail: { - int j; - StrAccum errMsg; - const char *zSep; - char *zErr; - - sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; - zSep = pIdx->nColumn>1 ? "columns " : "column "; - for(j=0; jnColumn; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - sqlite3StrAccumAppend(&errMsg, zSep, -1); - zSep = ", "; - sqlite3StrAccumAppend(&errMsg, zCol, -1); - } - sqlite3StrAccumAppend(&errMsg, - pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); - zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, onError, zErr, 0); - sqlite3DbFree(errMsg.db, zErr); - break; - } - case OE_Ignore: { - assert( seenReplace==0 ); - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - break; - } - default: { - Trigger *pTrigger = 0; - assert( onError==OE_Replace ); - sqlite3MultiWrite(pParse); - if( pParse->db->flags&SQLITE_RecTriggers ){ - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - } - sqlite3GenerateRowDelete( - pParse, pTab, baseCur, regR, 0, pTrigger, OE_Replace - ); - seenReplace = 1; - break; - } - } - sqlite3VdbeJumpHere(v, j3); - sqlite3ReleaseTempReg(pParse, regR); - } - - if( pbMayReplace ){ - *pbMayReplace = seenReplace; - } +/* +** Return the collating function associated with a function. +*/ +static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ + return context->pColl; } /* -** This routine generates code to finish the INSERT or UPDATE operation -** that was started by a prior call to sqlite3GenerateConstraintChecks. -** A consecutive range of registers starting at regRowid contains the -** rowid and the content to be inserted. -** -** The arguments to this routine should be the same as the first six -** arguments to sqlite3GenerateConstraintChecks. +** Implementation of the non-aggregate min() and max() functions */ -SQLITE_PRIVATE void sqlite3CompleteInsertion( - Parse *pParse, /* The parser context */ - Table *pTab, /* the table into which we are inserting */ - int baseCur, /* Index of a read/write cursor pointing at pTab */ - int regRowid, /* Range of content */ - int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int isUpdate, /* True for UPDATE, False for INSERT */ - int appendBias, /* True if this is likely to be an append */ - int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ +static void minmaxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ int i; - Vdbe *v; - int nIdx; - Index *pIdx; - u8 pik_flags; - int regData; - int regRec; + int mask; /* 0 for min() or 0xffffffff for max() */ + int iBest; + CollSeq *pColl; - v = sqlite3GetVdbe(pParse); - assert( v!=0 ); - assert( pTab->pSelect==0 ); /* This table is not a VIEW */ - for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} - for(i=nIdx-1; i>=0; i--){ - if( aRegIdx[i]==0 ) continue; - sqlite3VdbeAddOp2(v, OP_IdxInsert, baseCur+i+1, aRegIdx[i]); - if( useSeekResult ){ - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + assert( argc>1 ); + mask = sqlite3_user_data(context)==0 ? 0 : -1; + pColl = sqlite3GetFuncCollSeq(context); + assert( pColl ); + assert( mask==-1 || mask==0 ); + iBest = 0; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + for(i=1; i=0 ){ + testcase( mask==0 ); + iBest = i; } } - regData = regRowid + 1; - regRec = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - sqlite3TableAffinityStr(v, pTab); - sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); - if( pParse->nested ){ - pik_flags = 0; - }else{ - pik_flags = OPFLAG_NCHANGE; - pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); - } - if( appendBias ){ - pik_flags |= OPFLAG_APPEND; - } - if( useSeekResult ){ - pik_flags |= OPFLAG_USESEEKRESULT; - } - sqlite3VdbeAddOp3(v, OP_Insert, baseCur, regRec, regRowid); - if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC); - } - sqlite3VdbeChangeP5(v, pik_flags); + sqlite3_result_value(context, argv[iBest]); } /* -** Generate code that will open cursors for a table and for all -** indices of that table. The "baseCur" parameter is the cursor number used -** for the table. Indices are opened on subsequent cursors. -** -** Return the number of indices on the table. +** Return the type of the argument. */ -SQLITE_PRIVATE int sqlite3OpenTableAndIndices( - Parse *pParse, /* Parsing context */ - Table *pTab, /* Table to be opened */ - int baseCur, /* Cursor number assigned to the table */ - int op /* OP_OpenRead or OP_OpenWrite */ +static void typeofFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv ){ - int i; - int iDb; - Index *pIdx; - Vdbe *v; - - if( IsVirtual(pTab) ) return 0; - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - v = sqlite3GetVdbe(pParse); - assert( v!=0 ); - sqlite3OpenTable(pParse, baseCur, iDb, pTab, op); - for(i=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); - assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp4(v, op, i+baseCur, pIdx->tnum, iDb, - (char*)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIdx->zName)); - } - if( pParse->nTabnTab = baseCur+i; + const char *z = 0; + UNUSED_PARAMETER(NotUsed); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: z = "integer"; break; + case SQLITE_TEXT: z = "text"; break; + case SQLITE_FLOAT: z = "real"; break; + case SQLITE_BLOB: z = "blob"; break; + default: z = "null"; break; } - return i-1; + sqlite3_result_text(context, z, -1, SQLITE_STATIC); } -#ifdef SQLITE_TEST /* -** The following global variable is incremented whenever the -** transfer optimization is used. This is used for testing -** purposes only - to make sure the transfer optimization really -** is happening when it is suppose to. +** Implementation of the length() function */ -SQLITE_API int sqlite3_xferopt_count; -#endif /* SQLITE_TEST */ - +static void lengthFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int len; -#ifndef SQLITE_OMIT_XFER_OPT -/* -** Check to collation names to see if they are compatible. -*/ -static int xferCompatibleCollation(const char *z1, const char *z2){ - if( z1==0 ){ - return z2==0; - } - if( z2==0 ){ - return 0; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_BLOB: + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + break; + } + case SQLITE_TEXT: { + const unsigned char *z = sqlite3_value_text(argv[0]); + if( z==0 ) return; + len = 0; + while( *z ){ + len++; + SQLITE_SKIP_UTF8(z); + } + sqlite3_result_int(context, len); + break; + } + default: { + sqlite3_result_null(context); + break; + } } - return sqlite3StrICmp(z1, z2)==0; } - /* -** Check to see if index pSrc is compatible as a source of data -** for index pDest in an insert transfer optimization. The rules -** for a compatible index: +** Implementation of the abs() function. ** -** * The index is over the same set of columns -** * The same DESC and ASC markings occurs on all columns -** * The same onError processing (OE_Abort, OE_Ignore, etc) -** * The same collating sequence on each column +** IMP: R-23979-26855 The abs(X) function returns the absolute value of +** the numeric argument X. */ -static int xferCompatibleIndex(Index *pDest, Index *pSrc){ - int i; - assert( pDest && pSrc ); - assert( pDest->pTable!=pSrc->pTable ); - if( pDest->nColumn!=pSrc->nColumn ){ - return 0; /* Different number of columns */ - } - if( pDest->onError!=pSrc->onError ){ - return 0; /* Different conflict resolution strategies */ - } - for(i=0; inColumn; i++){ - if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ - return 0; /* Different columns indexed */ +static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal<0 ){ + if( (iVal<<1)==0 ){ + /* IMP: R-35460-15084 If X is the integer -9223372036854775807 then + ** abs(X) throws an integer overflow error since there is no + ** equivalent positive 64-bit two complement value. */ + sqlite3_result_error(context, "integer overflow", -1); + return; + } + iVal = -iVal; + } + sqlite3_result_int64(context, iVal); + break; } - if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ - return 0; /* Different sort orders */ + case SQLITE_NULL: { + /* IMP: R-37434-19929 Abs(X) returns NULL if X is NULL. */ + sqlite3_result_null(context); + break; } - if( !xferCompatibleCollation(pSrc->azColl[i],pDest->azColl[i]) ){ - return 0; /* Different collating sequences */ + default: { + /* Because sqlite3_value_double() returns 0.0 if the argument is not + ** something that can be converted into a number, we have: + ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that + ** cannot be converted to a numeric value. + */ + double rVal = sqlite3_value_double(argv[0]); + if( rVal<0 ) rVal = -rVal; + sqlite3_result_double(context, rVal); + break; } } - - /* If no test above fails then the indices must be compatible */ - return 1; } /* -** Attempt the transfer optimization on INSERTs of the form -** -** INSERT INTO tab1 SELECT * FROM tab2; -** -** This optimization is only attempted if +** Implementation of the substr() function. ** -** (1) tab1 and tab2 have identical schemas including all the -** same indices and constraints +** substr(x,p1,p2) returns p2 characters of x[] beginning with p1. +** p1 is 1-indexed. So substr(x,1,1) returns the first character +** of x. If x is text, then we actually count UTF-8 characters. +** If x is a blob, then we count bytes. ** -** (2) tab1 and tab2 are different tables +** If p1 is negative, then we begin abs(p1) from the end of x[]. ** -** (3) There must be no triggers on tab1 -** -** (4) The result set of the SELECT statement is "*" -** -** (5) The SELECT statement has no WHERE, HAVING, ORDER BY, GROUP BY, -** or LIMIT clause. -** -** (6) The SELECT statement is a simple (not a compound) select that -** contains only tab2 in its FROM clause -** -** This method for implementing the INSERT transfers raw records from -** tab2 over to tab1. The columns are not decoded. Raw records from -** the indices of tab2 are transfered to tab1 as well. In so doing, -** the resulting tab1 has much less fragmentation. -** -** This routine returns TRUE if the optimization is attempted. If any -** of the conditions above fail so that the optimization should not -** be attempted, then this routine returns FALSE. +** If p2 is negative, return the p2 characters preceeding p1. */ -static int xferOptimization( - Parse *pParse, /* Parser context */ - Table *pDest, /* The table we are inserting into */ - Select *pSelect, /* A SELECT statement to use as the data source */ - int onError, /* How to handle constraint errors */ - int iDbDest /* The database of pDest */ +static void substrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ - ExprList *pEList; /* The result set of the SELECT */ - Table *pSrc; /* The table in the FROM clause of SELECT */ - Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ - struct SrcList_item *pItem; /* An element of pSelect->pSrc */ - int i; /* Loop counter */ - int iDbSrc; /* The database of pSrc */ - int iSrc, iDest; /* Cursors from source and destination */ - int addr1, addr2; /* Loop addresses */ - int emptyDestTest; /* Address of test for empty pDest */ - int emptySrcTest; /* Address of test for empty pSrc */ - Vdbe *v; /* The VDBE we are building */ - KeyInfo *pKey; /* Key information for an index */ - int regAutoinc; /* Memory register used by AUTOINC */ - int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ - int regData, regRowid; /* Registers holding data and rowid */ - - if( pSelect==0 ){ - return 0; /* Must be of the form INSERT INTO ... SELECT ... */ - } - if( sqlite3TriggerList(pParse, pDest) ){ - return 0; /* tab1 must not have triggers */ - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pDest->tabFlags & TF_Virtual ){ - return 0; /* tab1 must not be a virtual table */ - } -#endif - if( onError==OE_Default ){ - onError = OE_Abort; - } - if( onError!=OE_Abort && onError!=OE_Rollback ){ - return 0; /* Cannot do OR REPLACE or OR IGNORE or OR FAIL */ - } - assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ - if( pSelect->pSrc->nSrc!=1 ){ - return 0; /* FROM clause must have exactly one term */ - } - if( pSelect->pSrc->a[0].pSelect ){ - return 0; /* FROM clause cannot contain a subquery */ - } - if( pSelect->pWhere ){ - return 0; /* SELECT may not have a WHERE clause */ - } - if( pSelect->pOrderBy ){ - return 0; /* SELECT may not have an ORDER BY clause */ - } - /* Do not need to test for a HAVING clause. If HAVING is present but - ** there is no ORDER BY, we will get an error. */ - if( pSelect->pGroupBy ){ - return 0; /* SELECT may not have a GROUP BY clause */ - } - if( pSelect->pLimit ){ - return 0; /* SELECT may not have a LIMIT clause */ - } - assert( pSelect->pOffset==0 ); /* Must be so if pLimit==0 */ - if( pSelect->pPrior ){ - return 0; /* SELECT may not be a compound query */ - } - if( pSelect->selFlags & SF_Distinct ){ - return 0; /* SELECT may not be DISTINCT */ - } - pEList = pSelect->pEList; - assert( pEList!=0 ); - if( pEList->nExpr!=1 ){ - return 0; /* The result set must have exactly one column */ - } - assert( pEList->a[0].pExpr ); - if( pEList->a[0].pExpr->op!=TK_ALL ){ - return 0; /* The result set must be the special operator "*" */ - } + const unsigned char *z; + const unsigned char *z2; + int len; + int p0type; + i64 p1, p2; + int negP2 = 0; - /* At this point we have established that the statement is of the - ** correct syntactic form to participate in this optimization. Now - ** we have to check the semantics. - */ - pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); - if( pSrc==0 ){ - return 0; /* FROM clause does not contain a real table */ - } - if( pSrc==pDest ){ - return 0; /* tab1 and tab2 may not be the same table */ - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pSrc->tabFlags & TF_Virtual ){ - return 0; /* tab2 must not be a virtual table */ - } -#endif - if( pSrc->pSelect ){ - return 0; /* tab2 may not be a view */ - } - if( pDest->nCol!=pSrc->nCol ){ - return 0; /* Number of columns must be the same in tab1 and tab2 */ + assert( argc==3 || argc==2 ); + if( sqlite3_value_type(argv[1])==SQLITE_NULL + || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) + ){ + return; } - if( pDest->iPKey!=pSrc->iPKey ){ - return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ + p0type = sqlite3_value_type(argv[0]); + p1 = sqlite3_value_int(argv[1]); + if( p0type==SQLITE_BLOB ){ + len = sqlite3_value_bytes(argv[0]); + z = sqlite3_value_blob(argv[0]); + if( z==0 ) return; + assert( len==sqlite3_value_bytes(argv[0]) ); + }else{ + z = sqlite3_value_text(argv[0]); + if( z==0 ) return; + len = 0; + if( p1<0 ){ + for(z2=z; *z2; len++){ + SQLITE_SKIP_UTF8(z2); + } + } } - for(i=0; inCol; i++){ - if( pDest->aCol[i].affinity!=pSrc->aCol[i].affinity ){ - return 0; /* Affinity must be the same on all columns */ + if( argc==3 ){ + p2 = sqlite3_value_int(argv[2]); + if( p2<0 ){ + p2 = -p2; + negP2 = 1; } - if( !xferCompatibleCollation(pDest->aCol[i].zColl, pSrc->aCol[i].zColl) ){ - return 0; /* Collating sequence must be the same on all columns */ + }else{ + p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; + } + if( p1<0 ){ + p1 += len; + if( p1<0 ){ + p2 += p1; + if( p2<0 ) p2 = 0; + p1 = 0; } - if( pDest->aCol[i].notNull && !pSrc->aCol[i].notNull ){ - return 0; /* tab2 must be NOT NULL if tab1 is */ + }else if( p1>0 ){ + p1--; + }else if( p2>0 ){ + p2--; + } + if( negP2 ){ + p1 -= p2; + if( p1<0 ){ + p2 += p1; + p1 = 0; } } - for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - if( pDestIdx->onError!=OE_None ){ - destHasUniqueIdx = 1; + assert( p1>=0 && p2>=0 ); + if( p0type!=SQLITE_BLOB ){ + while( *z && p1 ){ + SQLITE_SKIP_UTF8(z); + p1--; } - for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ - if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + for(z2=z; *z2 && p2; p2--){ + SQLITE_SKIP_UTF8(z2); } - if( pSrcIdx==0 ){ - return 0; /* pDestIdx has no corresponding index in pSrc */ + sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); + }else{ + if( p1+p2>len ){ + p2 = len-p1; + if( p2<0 ) p2 = 0; } + sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); } -#ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ - return 0; /* Tables have different CHECK constraints. Ticket #2252 */ - } -#endif +} - /* If we get this far, it means either: - ** - ** * We can always do the transfer if the table contains an - ** an integer primary key - ** - ** * We can conditionally do the transfer if the destination - ** table is empty. +/* +** Implementation of the round() function +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + int n = 0; + double r; + char *zBuf; + assert( argc==1 || argc==2 ); + if( argc==2 ){ + if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; + n = sqlite3_value_int(argv[1]); + if( n>30 ) n = 30; + if( n<0 ) n = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + r = sqlite3_value_double(argv[0]); + /* If Y==0 and X will fit in a 64-bit int, + ** handle the rounding directly, + ** otherwise use printf. */ -#ifdef SQLITE_TEST - sqlite3_xferopt_count++; -#endif - iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); - v = sqlite3GetVdbe(pParse); - sqlite3CodeVerifySchema(pParse, iDbSrc); - iSrc = pParse->nTab++; - iDest = pParse->nTab++; - regAutoinc = autoIncBegin(pParse, iDbDest, pDest); - sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); - if( (pDest->iPKey<0 && pDest->pIndex!=0) || destHasUniqueIdx ){ - /* If tables do not have an INTEGER PRIMARY KEY and there - ** are indices to be copied and the destination is not empty, - ** we have to disallow the transfer optimization because the - ** the rowids might change which will mess up indexing. - ** - ** Or if the destination has a UNIQUE index and is not empty, - ** we also disallow the transfer optimization because we cannot - ** insure that all entries in the union of DEST and SRC will be - ** unique. - */ - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); - emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - sqlite3VdbeJumpHere(v, addr1); + if( n==0 && r>=0 && riPKey>=0 ){ - addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); - addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - sqlite3HaltConstraint( - pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); - sqlite3VdbeJumpHere(v, addr2); - autoIncStep(pParse, regAutoinc, regRowid); - }else if( pDest->pIndex==0 ){ - addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); + sqlite3_result_double(context, r); +} +#endif + +/* +** Allocate nByte bytes of space using sqlite3_malloc(). If the +** allocation fails, call sqlite3_result_error_nomem() to notify +** the database handle that malloc() has failed and return NULL. +** If nByte is larger than the maximum string or blob length, then +** raise an SQLITE_TOOBIG exception and return NULL. +*/ +static void *contextMalloc(sqlite3_context *context, i64 nByte){ + char *z; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( nByte>0 ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + z = 0; }else{ - addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); - assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + z = sqlite3Malloc((int)nByte); + if( !z ){ + sqlite3_result_error_nomem(context); + } } - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); - sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); - sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); - sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); - for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ - if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + return z; +} + +/* +** Implementation of the upper() and lower() SQL functions. +*/ +static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + char *z1; + const char *z2; + int i, n; + UNUSED_PARAMETER(argc); + z2 = (char*)sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + /* Verify that the call to _bytes() does not invalidate the _text() pointer */ + assert( z2==(char*)sqlite3_value_text(argv[0]) ); + if( z2 ){ + z1 = contextMalloc(context, ((i64)n)+1); + if( z1 ){ + memcpy(z1, z2, n+1); + for(i=0; z1[i]; i++){ + z1[i] = (char)sqlite3Toupper(z1[i]); + } + sqlite3_result_text(context, z1, -1, sqlite3_free); } - assert( pSrcIdx ); - sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); - sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); - pKey = sqlite3IndexKeyinfo(pParse, pSrcIdx); - sqlite3VdbeAddOp4(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc, - (char*)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pSrcIdx->zName)); - pKey = sqlite3IndexKeyinfo(pParse, pDestIdx); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest, - (char*)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pDestIdx->zName)); - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); - sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); - sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); - sqlite3VdbeJumpHere(v, addr1); } - sqlite3VdbeJumpHere(v, emptySrcTest); - sqlite3ReleaseTempReg(pParse, regRowid); - sqlite3ReleaseTempReg(pParse, regData); - sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); - sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); - if( emptyDestTest ){ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); - sqlite3VdbeJumpHere(v, emptyDestTest); - sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); - return 0; - }else{ - return 1; +} +static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + u8 *z1; + const char *z2; + int i, n; + UNUSED_PARAMETER(argc); + z2 = (char*)sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + /* Verify that the call to _bytes() does not invalidate the _text() pointer */ + assert( z2==(char*)sqlite3_value_text(argv[0]) ); + if( z2 ){ + z1 = contextMalloc(context, ((i64)n)+1); + if( z1 ){ + memcpy(z1, z2, n+1); + for(i=0; z1[i]; i++){ + z1[i] = sqlite3Tolower(z1[i]); + } + sqlite3_result_text(context, (char *)z1, -1, sqlite3_free); + } } } -#endif /* SQLITE_OMIT_XFER_OPT */ -/************** End of insert.c **********************************************/ -/************** Begin file legacy.c ******************************************/ + +#if 0 /* This function is never used. */ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Main file for the SQLite library. The routines in this file -** implement the programmer interface to the library. Routines in -** other files are for internal use by SQLite and should not be -** accessed by users of the library. +** The COALESCE() and IFNULL() functions used to be implemented as shown +** here. But now they are implemented as VDBE code so that unused arguments +** do not have to be computed. This legacy implementation is retained as +** comment. +*/ +/* +** Implementation of the IFNULL(), NVL(), and COALESCE() functions. +** All three do the same thing. They return the first non-NULL +** argument. */ +static void ifnullFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + for(i=0; imutex); - sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ - int nCol; - char **azVals = 0; +/* +** Implementation of the changes() SQL function. +** +** IMP: R-62073-11209 The changes() SQL function is a wrapper +** around the sqlite3_changes() C/C++ function and hence follows the same +** rules for counting changes. +*/ +static void changes( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + sqlite3 *db = sqlite3_context_db_handle(context); + UNUSED_PARAMETER2(NotUsed, NotUsed2); + sqlite3_result_int(context, sqlite3_changes(db)); +} - pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); - assert( rc==SQLITE_OK || pStmt==0 ); - if( rc!=SQLITE_OK ){ - continue; - } - if( !pStmt ){ - /* this happens for a comment or white-space */ - zSql = zLeftover; - continue; - } +/* +** Implementation of the total_changes() SQL function. The return value is +** the same as the sqlite3_total_changes() API function. +*/ +static void total_changes( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + sqlite3 *db = sqlite3_context_db_handle(context); + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-52756-41993 This function is a wrapper around the + ** sqlite3_total_changes() C/C++ interface. */ + sqlite3_result_int(context, sqlite3_total_changes(db)); +} - callbackIsInit = 0; - nCol = sqlite3_column_count(pStmt); +/* +** A structure defining how to do GLOB-style comparisons. +*/ +struct compareInfo { + u8 matchAll; + u8 matchOne; + u8 matchSet; + u8 noCase; +}; - while( 1 ){ - int i; - rc = sqlite3_step(pStmt); +/* +** For LIKE and GLOB matching on EBCDIC machines, assume that every +** character is exactly one byte in size. Also, all characters are +** able to participate in upper-case-to-lower-case mappings in EBCDIC +** whereas only characters less than 0x80 do in ASCII. +*/ +#if defined(SQLITE_EBCDIC) +# define sqlite3Utf8Read(A,C) (*(A++)) +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] +#else +# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } +#endif - /* Invoke the callback function if required */ - if( xCallback && (SQLITE_ROW==rc || - (SQLITE_DONE==rc && !callbackIsInit - && db->flags&SQLITE_NullCallback)) ){ - if( !callbackIsInit ){ - azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); - if( azCols==0 ){ - goto exec_out; +static const struct compareInfo globInfo = { '*', '?', '[', 0 }; +/* The correct SQL-92 behavior is for the LIKE operator to ignore +** case. Thus 'a' LIKE 'A' would be true. */ +static const struct compareInfo likeInfoNorm = { '%', '_', 0, 1 }; +/* If SQLITE_CASE_SENSITIVE_LIKE is defined, then the LIKE operator +** is case sensitive causing 'a' LIKE 'A' to be false */ +static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; + +/* +** Compare two UTF-8 strings for equality where the first string can +** potentially be a "glob" expression. Return true (1) if they +** are the same and false (0) if they are different. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** With the [...] and [^...] matching, a ']' character can be included +** in the list by making it the first character after '[' or '^'. A +** range of characters can be specified using '-'. Example: +** "[a-z]" matches any single lower-case letter. To match a '-', make +** it the last character in the list. +** +** This routine is usually quick, but can be N**2 in the worst case. +** +** Hints: to match '*' or '?', put them in "[]". Like this: +** +** abc[*]xyz Matches "abc*xyz" only +*/ +static int patternCompare( + const u8 *zPattern, /* The glob pattern */ + const u8 *zString, /* The string to compare against the glob */ + const struct compareInfo *pInfo, /* Information about how to do the compare */ + u32 esc /* The escape character */ +){ + u32 c, c2; + int invert; + int seen; + u8 matchOne = pInfo->matchOne; + u8 matchAll = pInfo->matchAll; + u8 matchSet = pInfo->matchSet; + u8 noCase = pInfo->noCase; + int prevEscape = 0; /* True if the previous character was 'escape' */ + + while( (c = sqlite3Utf8Read(zPattern,&zPattern))!=0 ){ + if( !prevEscape && c==matchAll ){ + while( (c=sqlite3Utf8Read(zPattern,&zPattern)) == matchAll + || c == matchOne ){ + if( c==matchOne && sqlite3Utf8Read(zString, &zString)==0 ){ + return 0; + } + } + if( c==0 ){ + return 1; + }else if( c==esc ){ + c = sqlite3Utf8Read(zPattern, &zPattern); + if( c==0 ){ + return 0; + } + }else if( c==matchSet ){ + assert( esc==0 ); /* This is GLOB, not LIKE */ + assert( matchSet<0x80 ); /* '[' is a single-byte character */ + while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + SQLITE_SKIP_UTF8(zString); + } + return *zString!=0; + } + while( (c2 = sqlite3Utf8Read(zString,&zString))!=0 ){ + if( noCase ){ + GlogUpperToLower(c2); + GlogUpperToLower(c); + while( c2 != 0 && c2 != c ){ + c2 = sqlite3Utf8Read(zString, &zString); + GlogUpperToLower(c2); } - for(i=0; imallocFailed = 1; - goto exec_out; - } + if( c2==0 ) return 0; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + } + return 0; + }else if( !prevEscape && c==matchOne ){ + if( sqlite3Utf8Read(zString, &zString)==0 ){ + return 0; + } + }else if( c==matchSet ){ + u32 prior_c = 0; + assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ + seen = 0; + invert = 0; + c = sqlite3Utf8Read(zString, &zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(zPattern, &zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(zPattern, &zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(zPattern, &zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(zPattern, &zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; } + prior_c = c2; } - if( xCallback(pArg, nCol, azVals, azCols) ){ - rc = SQLITE_ABORT; - sqlite3VdbeFinalize((Vdbe *)pStmt); - pStmt = 0; - sqlite3Error(db, SQLITE_ABORT, 0); - goto exec_out; - } + c2 = sqlite3Utf8Read(zPattern, &zPattern); } - - if( rc!=SQLITE_ROW ){ - rc = sqlite3VdbeFinalize((Vdbe *)pStmt); - pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } - break; + if( c2==0 || (seen ^ invert)==0 ){ + return 0; } - } - - sqlite3DbFree(db, azCols); - azCols = 0; - } - -exec_out: - if( pStmt ) sqlite3VdbeFinalize((Vdbe *)pStmt); - sqlite3DbFree(db, azCols); - - rc = sqlite3ApiExit(db, rc); - if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ - int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); - *pzErrMsg = sqlite3Malloc(nErrMsg); - if( *pzErrMsg ){ - memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); + }else if( esc==c && !prevEscape ){ + prevEscape = 1; }else{ - rc = SQLITE_NOMEM; - sqlite3Error(db, SQLITE_NOMEM, 0); + c2 = sqlite3Utf8Read(zString, &zString); + if( noCase ){ + GlogUpperToLower(c); + GlogUpperToLower(c2); + } + if( c!=c2 ){ + return 0; + } + prevEscape = 0; } - }else if( pzErrMsg ){ - *pzErrMsg = 0; } - - assert( (rc&db->errMask)==rc ); - sqlite3_mutex_leave(db->mutex); - return rc; + return *zString==0; } -/************** End of legacy.c **********************************************/ -/************** Begin file loadext.c *****************************************/ /* -** 2006 June 7 +** Count the number of times that the LIKE operator (or GLOB which is +** just a variation of LIKE) gets called. This is used for testing +** only. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_like_count = 0; +#endif + + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** A LIKE B ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** is implemented as like(B,A). ** -************************************************************************* -** This file contains code used to dynamically load extensions into -** the SQLite library. +** This same function (with a different compareInfo structure) computes +** the GLOB operator. */ +static void likeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA, *zB; + u32 escape = 0; + int nPat; + sqlite3 *db = sqlite3_context_db_handle(context); -#ifndef SQLITE_CORE - #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + nPat = sqlite3_value_bytes(argv[0]); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]+1 ); + if( nPat > db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + const unsigned char *zEsc = sqlite3_value_text(argv[2]); + if( zEsc==0 ) return; + if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + escape = sqlite3Utf8Read(zEsc, &zEsc); + } + if( zA && zB ){ + struct compareInfo *pInfo = sqlite3_user_data(context); +#ifdef SQLITE_TEST + sqlite3_like_count++; #endif -/************** Include sqlite3ext.h in the middle of loadext.c **************/ -/************** Begin file sqlite3ext.h **************************************/ + + sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); + } +} + /* -** 2006 June 7 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This header file defines the SQLite interface for use by -** shared libraries that want to be imported as extensions into -** an SQLite instance. Shared libraries that intend to be loaded -** as extensions by SQLite should #include this file instead of -** sqlite3.h. +** Implementation of the NULLIF(x,y) function. The result is the first +** argument if the arguments are different. The result is NULL if the +** arguments are equal to each other. */ -#ifndef _SQLITE3EXT_H_ -#define _SQLITE3EXT_H_ +static void nullifFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + UNUSED_PARAMETER(NotUsed); + if( sqlite3MemCompare(argv[0], argv[1], pColl)!=0 ){ + sqlite3_result_value(context, argv[0]); + } +} -typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** Implementation of the sqlite_version() function. The result is the version +** of the SQLite library that is running. +*/ +static void versionFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-48699-48617 This function is an SQL wrapper around the + ** sqlite3_libversion() C-interface. */ + sqlite3_result_text(context, sqlite3_libversion(), -1, SQLITE_STATIC); +} /* -** The following structure holds pointers to all of the SQLite API -** routines. +** Implementation of the sqlite_source_id() function. The result is a string +** that identifies the particular version of the source code used to build +** SQLite. +*/ +static void sourceidFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-24470-31136 This function is an SQL wrapper around the + ** sqlite3_sourceid() C interface. */ + sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_log() function. This is a wrapper around +** sqlite3_log(). The return value is NULL. The function exists purely for +** its side-effects. +*/ +static void errlogFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(context); + sqlite3_log(sqlite3_value_int(argv[0]), "%s", sqlite3_value_text(argv[1])); +} + +/* +** Implementation of the sqlite_compileoption_used() function. +** The result is an integer that identifies if the compiler option +** was used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptionusedFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zOptName; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ + if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); + } +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* +** Implementation of the sqlite_compileoption_get() function. +** The result is a string that identifies the compiler options +** used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptiongetFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ + n = sqlite3_value_int(argv[0]); + sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* Array for converting from half-bytes (nybbles) into ASCII hex +** digits. */ +static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' +}; + +/* +** EXPERIMENTAL - This is not an official function. The interface may +** change. This function may disappear. Do not write code that depends +** on this function. ** -** WARNING: In order to maintain backwards compatibility, add new -** interfaces to the end of this structure only. If you insert new -** interfaces in the middle of this structure, then older different -** versions of SQLite will not be able to load each others' shared -** libraries! +** Implementation of the QUOTE() function. This function takes a single +** argument. If the argument is numeric, the return value is the same as +** the argument. If the argument is NULL, the return value is the string +** "NULL". Otherwise, the argument is enclosed in single quotes with +** single-quote escapes. */ -struct sqlite3_api_routines { - void * (*aggregate_context)(sqlite3_context*,int nBytes); - int (*aggregate_count)(sqlite3_context*); - int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); - int (*bind_double)(sqlite3_stmt*,int,double); - int (*bind_int)(sqlite3_stmt*,int,int); - int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); - int (*bind_null)(sqlite3_stmt*,int); - int (*bind_parameter_count)(sqlite3_stmt*); - int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); - const char * (*bind_parameter_name)(sqlite3_stmt*,int); - int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); - int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); - int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); - int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); - int (*busy_timeout)(sqlite3*,int ms); - int (*changes)(sqlite3*); - int (*close)(sqlite3*); - int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const char*)); - int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const void*)); - const void * (*column_blob)(sqlite3_stmt*,int iCol); - int (*column_bytes)(sqlite3_stmt*,int iCol); - int (*column_bytes16)(sqlite3_stmt*,int iCol); - int (*column_count)(sqlite3_stmt*pStmt); - const char * (*column_database_name)(sqlite3_stmt*,int); - const void * (*column_database_name16)(sqlite3_stmt*,int); - const char * (*column_decltype)(sqlite3_stmt*,int i); - const void * (*column_decltype16)(sqlite3_stmt*,int); - double (*column_double)(sqlite3_stmt*,int iCol); - int (*column_int)(sqlite3_stmt*,int iCol); - sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); - const char * (*column_name)(sqlite3_stmt*,int); - const void * (*column_name16)(sqlite3_stmt*,int); - const char * (*column_origin_name)(sqlite3_stmt*,int); - const void * (*column_origin_name16)(sqlite3_stmt*,int); - const char * (*column_table_name)(sqlite3_stmt*,int); - const void * (*column_table_name16)(sqlite3_stmt*,int); - const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); - const void * (*column_text16)(sqlite3_stmt*,int iCol); - int (*column_type)(sqlite3_stmt*,int iCol); - sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); - void * (*commit_hook)(sqlite3*,int(*)(void*),void*); - int (*complete)(const char*sql); - int (*complete16)(const void*sql); - int (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); - int (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); - int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); - int (*data_count)(sqlite3_stmt*pStmt); - sqlite3 * (*db_handle)(sqlite3_stmt*); - int (*declare_vtab)(sqlite3*,const char*); - int (*enable_shared_cache)(int); - int (*errcode)(sqlite3*db); - const char * (*errmsg)(sqlite3*); - const void * (*errmsg16)(sqlite3*); - int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); - int (*expired)(sqlite3_stmt*); - int (*finalize)(sqlite3_stmt*pStmt); - void (*free)(void*); - void (*free_table)(char**result); - int (*get_autocommit)(sqlite3*); - void * (*get_auxdata)(sqlite3_context*,int); - int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); - int (*global_recover)(void); - void (*interruptx)(sqlite3*); - sqlite_int64 (*last_insert_rowid)(sqlite3*); - const char * (*libversion)(void); - int (*libversion_number)(void); - void *(*malloc)(int); - char * (*mprintf)(const char*,...); - int (*open)(const char*,sqlite3**); - int (*open16)(const void*,sqlite3**); - int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); - int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); - void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); - void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); - void *(*realloc)(void*,int); - int (*reset)(sqlite3_stmt*pStmt); - void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); - void (*result_double)(sqlite3_context*,double); - void (*result_error)(sqlite3_context*,const char*,int); - void (*result_error16)(sqlite3_context*,const void*,int); - void (*result_int)(sqlite3_context*,int); - void (*result_int64)(sqlite3_context*,sqlite_int64); - void (*result_null)(sqlite3_context*); - void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); - void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); - void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); - void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); - void (*result_value)(sqlite3_context*,sqlite3_value*); - void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); - int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,const char*,const char*),void*); - void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); - char * (*snprintf)(int,char*,const char*,...); - int (*step)(sqlite3_stmt*); - int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,char const**,char const**,int*,int*,int*); - void (*thread_cleanup)(void); - int (*total_changes)(sqlite3*); - void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); - int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); - void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,sqlite_int64),void*); - void * (*user_data)(sqlite3_context*); - const void * (*value_blob)(sqlite3_value*); - int (*value_bytes)(sqlite3_value*); - int (*value_bytes16)(sqlite3_value*); - double (*value_double)(sqlite3_value*); - int (*value_int)(sqlite3_value*); - sqlite_int64 (*value_int64)(sqlite3_value*); - int (*value_numeric_type)(sqlite3_value*); - const unsigned char * (*value_text)(sqlite3_value*); - const void * (*value_text16)(sqlite3_value*); - const void * (*value_text16be)(sqlite3_value*); - const void * (*value_text16le)(sqlite3_value*); - int (*value_type)(sqlite3_value*); - char *(*vmprintf)(const char*,va_list); - /* Added ??? */ - int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); - /* Added by 3.3.13 */ - int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); - int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); - int (*clear_bindings)(sqlite3_stmt*); - /* Added by 3.4.1 */ - int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *)); - /* Added by 3.5.0 */ - int (*bind_zeroblob)(sqlite3_stmt*,int,int); - int (*blob_bytes)(sqlite3_blob*); - int (*blob_close)(sqlite3_blob*); - int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**); - int (*blob_read)(sqlite3_blob*,void*,int,int); - int (*blob_write)(sqlite3_blob*,const void*,int,int); - int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*)); - int (*file_control)(sqlite3*,const char*,int,void*); - sqlite3_int64 (*memory_highwater)(int); - sqlite3_int64 (*memory_used)(void); - sqlite3_mutex *(*mutex_alloc)(int); - void (*mutex_enter)(sqlite3_mutex*); - void (*mutex_free)(sqlite3_mutex*); - void (*mutex_leave)(sqlite3_mutex*); - int (*mutex_try)(sqlite3_mutex*); - int (*open_v2)(const char*,sqlite3**,int,const char*); - int (*release_memory)(int); - void (*result_error_nomem)(sqlite3_context*); - void (*result_error_toobig)(sqlite3_context*); - int (*sleep)(int); - void (*soft_heap_limit)(int); - sqlite3_vfs *(*vfs_find)(const char*); - int (*vfs_register)(sqlite3_vfs*,int); - int (*vfs_unregister)(sqlite3_vfs*); - int (*xthreadsafe)(void); - void (*result_zeroblob)(sqlite3_context*,int); - void (*result_error_code)(sqlite3_context*,int); - int (*test_control)(int, ...); - void (*randomness)(int,void*); - sqlite3 *(*context_db_handle)(sqlite3_context*); - int (*extended_result_codes)(sqlite3*,int); - int (*limit)(sqlite3*,int,int); - sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); - const char *(*sql)(sqlite3_stmt*); - int (*status)(int,int*,int*,int); - int (*backup_finish)(sqlite3_backup*); - sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); - int (*backup_pagecount)(sqlite3_backup*); - int (*backup_remaining)(sqlite3_backup*); - int (*backup_step)(sqlite3_backup*,int); - const char *(*compileoption_get)(int); - int (*compileoption_used)(const char*); - int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); - int (*db_config)(sqlite3*,int,...); - sqlite3_mutex *(*db_mutex)(sqlite3*); - int (*db_status)(sqlite3*,int,int*,int*,int); - int (*extended_errcode)(sqlite3*); - void (*log)(int,const char*,...); - sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); - const char *(*sourceid)(void); - int (*stmt_status)(sqlite3_stmt*,int,int); - int (*strnicmp)(const char*,const char*,int); - int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); - int (*wal_autocheckpoint)(sqlite3*,int); - int (*wal_checkpoint)(sqlite3*,const char*); - void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); -}; +static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + sqlite3_result_value(context, argv[0]); + break; + } + case SQLITE_BLOB: { + char *zText = 0; + char const *zBlob = sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); + if( zText ){ + int i; + for(i=0; i>4)&0x0F]; + zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; + } + zText[(nBlob*2)+2] = '\''; + zText[(nBlob*2)+3] = '\0'; + zText[0] = 'X'; + zText[1] = '\''; + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + break; + } + case SQLITE_TEXT: { + int i,j; + u64 n; + const unsigned char *zArg = sqlite3_value_text(argv[0]); + char *z; + + if( zArg==0 ) return; + for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } + z = contextMalloc(context, ((i64)i)+((i64)n)+3); + if( z ){ + z[0] = '\''; + for(i=0, j=1; zArg[i]; i++){ + z[j++] = zArg[i]; + if( zArg[i]=='\'' ){ + z[j++] = '\''; + } + } + z[j++] = '\''; + z[j] = 0; + sqlite3_result_text(context, z, j, sqlite3_free); + } + break; + } + default: { + assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); + sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); + break; + } + } +} /* -** The following macros redefine the API routines so that they are -** redirected throught the global sqlite3_api structure. -** -** This header file is also used by the loadext.c source file -** (part of the main SQLite library - not an extension) so that -** it can get access to the sqlite3_api_routines structure -** definition. But the main library does not want to redefine -** the API. So the redefinition macros are only valid if the -** SQLITE_CORE macros is undefined. +** The hex() function. Interpret the argument as a blob. Return +** a hexadecimal rendering as text. */ -#ifndef SQLITE_CORE -#define sqlite3_aggregate_context sqlite3_api->aggregate_context -#ifndef SQLITE_OMIT_DEPRECATED -#define sqlite3_aggregate_count sqlite3_api->aggregate_count -#endif -#define sqlite3_bind_blob sqlite3_api->bind_blob -#define sqlite3_bind_double sqlite3_api->bind_double -#define sqlite3_bind_int sqlite3_api->bind_int -#define sqlite3_bind_int64 sqlite3_api->bind_int64 -#define sqlite3_bind_null sqlite3_api->bind_null -#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count -#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index -#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name -#define sqlite3_bind_text sqlite3_api->bind_text -#define sqlite3_bind_text16 sqlite3_api->bind_text16 -#define sqlite3_bind_value sqlite3_api->bind_value -#define sqlite3_busy_handler sqlite3_api->busy_handler -#define sqlite3_busy_timeout sqlite3_api->busy_timeout -#define sqlite3_changes sqlite3_api->changes -#define sqlite3_close sqlite3_api->close -#define sqlite3_collation_needed sqlite3_api->collation_needed -#define sqlite3_collation_needed16 sqlite3_api->collation_needed16 -#define sqlite3_column_blob sqlite3_api->column_blob -#define sqlite3_column_bytes sqlite3_api->column_bytes -#define sqlite3_column_bytes16 sqlite3_api->column_bytes16 -#define sqlite3_column_count sqlite3_api->column_count -#define sqlite3_column_database_name sqlite3_api->column_database_name -#define sqlite3_column_database_name16 sqlite3_api->column_database_name16 -#define sqlite3_column_decltype sqlite3_api->column_decltype -#define sqlite3_column_decltype16 sqlite3_api->column_decltype16 -#define sqlite3_column_double sqlite3_api->column_double -#define sqlite3_column_int sqlite3_api->column_int -#define sqlite3_column_int64 sqlite3_api->column_int64 -#define sqlite3_column_name sqlite3_api->column_name -#define sqlite3_column_name16 sqlite3_api->column_name16 -#define sqlite3_column_origin_name sqlite3_api->column_origin_name -#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 -#define sqlite3_column_table_name sqlite3_api->column_table_name -#define sqlite3_column_table_name16 sqlite3_api->column_table_name16 -#define sqlite3_column_text sqlite3_api->column_text -#define sqlite3_column_text16 sqlite3_api->column_text16 -#define sqlite3_column_type sqlite3_api->column_type -#define sqlite3_column_value sqlite3_api->column_value -#define sqlite3_commit_hook sqlite3_api->commit_hook -#define sqlite3_complete sqlite3_api->complete -#define sqlite3_complete16 sqlite3_api->complete16 -#define sqlite3_create_collation sqlite3_api->create_collation -#define sqlite3_create_collation16 sqlite3_api->create_collation16 -#define sqlite3_create_function sqlite3_api->create_function -#define sqlite3_create_function16 sqlite3_api->create_function16 -#define sqlite3_create_module sqlite3_api->create_module -#define sqlite3_create_module_v2 sqlite3_api->create_module_v2 -#define sqlite3_data_count sqlite3_api->data_count -#define sqlite3_db_handle sqlite3_api->db_handle -#define sqlite3_declare_vtab sqlite3_api->declare_vtab -#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache -#define sqlite3_errcode sqlite3_api->errcode -#define sqlite3_errmsg sqlite3_api->errmsg -#define sqlite3_errmsg16 sqlite3_api->errmsg16 -#define sqlite3_exec sqlite3_api->exec -#ifndef SQLITE_OMIT_DEPRECATED -#define sqlite3_expired sqlite3_api->expired -#endif -#define sqlite3_finalize sqlite3_api->finalize -#define sqlite3_free sqlite3_api->free -#define sqlite3_free_table sqlite3_api->free_table -#define sqlite3_get_autocommit sqlite3_api->get_autocommit -#define sqlite3_get_auxdata sqlite3_api->get_auxdata -#define sqlite3_get_table sqlite3_api->get_table -#ifndef SQLITE_OMIT_DEPRECATED -#define sqlite3_global_recover sqlite3_api->global_recover -#endif -#define sqlite3_interrupt sqlite3_api->interruptx -#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid -#define sqlite3_libversion sqlite3_api->libversion -#define sqlite3_libversion_number sqlite3_api->libversion_number -#define sqlite3_malloc sqlite3_api->malloc -#define sqlite3_mprintf sqlite3_api->mprintf -#define sqlite3_open sqlite3_api->open -#define sqlite3_open16 sqlite3_api->open16 -#define sqlite3_prepare sqlite3_api->prepare -#define sqlite3_prepare16 sqlite3_api->prepare16 -#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 -#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 -#define sqlite3_profile sqlite3_api->profile -#define sqlite3_progress_handler sqlite3_api->progress_handler -#define sqlite3_realloc sqlite3_api->realloc -#define sqlite3_reset sqlite3_api->reset -#define sqlite3_result_blob sqlite3_api->result_blob -#define sqlite3_result_double sqlite3_api->result_double -#define sqlite3_result_error sqlite3_api->result_error -#define sqlite3_result_error16 sqlite3_api->result_error16 -#define sqlite3_result_int sqlite3_api->result_int -#define sqlite3_result_int64 sqlite3_api->result_int64 -#define sqlite3_result_null sqlite3_api->result_null -#define sqlite3_result_text sqlite3_api->result_text -#define sqlite3_result_text16 sqlite3_api->result_text16 -#define sqlite3_result_text16be sqlite3_api->result_text16be -#define sqlite3_result_text16le sqlite3_api->result_text16le -#define sqlite3_result_value sqlite3_api->result_value -#define sqlite3_rollback_hook sqlite3_api->rollback_hook -#define sqlite3_set_authorizer sqlite3_api->set_authorizer -#define sqlite3_set_auxdata sqlite3_api->set_auxdata -#define sqlite3_snprintf sqlite3_api->snprintf -#define sqlite3_step sqlite3_api->step -#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata -#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup -#define sqlite3_total_changes sqlite3_api->total_changes -#define sqlite3_trace sqlite3_api->trace -#ifndef SQLITE_OMIT_DEPRECATED -#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings -#endif -#define sqlite3_update_hook sqlite3_api->update_hook -#define sqlite3_user_data sqlite3_api->user_data -#define sqlite3_value_blob sqlite3_api->value_blob -#define sqlite3_value_bytes sqlite3_api->value_bytes -#define sqlite3_value_bytes16 sqlite3_api->value_bytes16 -#define sqlite3_value_double sqlite3_api->value_double -#define sqlite3_value_int sqlite3_api->value_int -#define sqlite3_value_int64 sqlite3_api->value_int64 -#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type -#define sqlite3_value_text sqlite3_api->value_text -#define sqlite3_value_text16 sqlite3_api->value_text16 -#define sqlite3_value_text16be sqlite3_api->value_text16be -#define sqlite3_value_text16le sqlite3_api->value_text16le -#define sqlite3_value_type sqlite3_api->value_type -#define sqlite3_vmprintf sqlite3_api->vmprintf -#define sqlite3_overload_function sqlite3_api->overload_function -#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 -#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 -#define sqlite3_clear_bindings sqlite3_api->clear_bindings -#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob -#define sqlite3_blob_bytes sqlite3_api->blob_bytes -#define sqlite3_blob_close sqlite3_api->blob_close -#define sqlite3_blob_open sqlite3_api->blob_open -#define sqlite3_blob_read sqlite3_api->blob_read -#define sqlite3_blob_write sqlite3_api->blob_write -#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 -#define sqlite3_file_control sqlite3_api->file_control -#define sqlite3_memory_highwater sqlite3_api->memory_highwater -#define sqlite3_memory_used sqlite3_api->memory_used -#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc -#define sqlite3_mutex_enter sqlite3_api->mutex_enter -#define sqlite3_mutex_free sqlite3_api->mutex_free -#define sqlite3_mutex_leave sqlite3_api->mutex_leave -#define sqlite3_mutex_try sqlite3_api->mutex_try -#define sqlite3_open_v2 sqlite3_api->open_v2 -#define sqlite3_release_memory sqlite3_api->release_memory -#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem -#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig -#define sqlite3_sleep sqlite3_api->sleep -#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit -#define sqlite3_vfs_find sqlite3_api->vfs_find -#define sqlite3_vfs_register sqlite3_api->vfs_register -#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister -#define sqlite3_threadsafe sqlite3_api->xthreadsafe -#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob -#define sqlite3_result_error_code sqlite3_api->result_error_code -#define sqlite3_test_control sqlite3_api->test_control -#define sqlite3_randomness sqlite3_api->randomness -#define sqlite3_context_db_handle sqlite3_api->context_db_handle -#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes -#define sqlite3_limit sqlite3_api->limit -#define sqlite3_next_stmt sqlite3_api->next_stmt -#define sqlite3_sql sqlite3_api->sql -#define sqlite3_status sqlite3_api->status -#define sqlite3_backup_finish sqlite3_api->backup_finish -#define sqlite3_backup_init sqlite3_api->backup_init -#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount -#define sqlite3_backup_remaining sqlite3_api->backup_remaining -#define sqlite3_backup_step sqlite3_api->backup_step -#define sqlite3_compileoption_get sqlite3_api->compileoption_get -#define sqlite3_compileoption_used sqlite3_api->compileoption_used -#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 -#define sqlite3_db_config sqlite3_api->db_config -#define sqlite3_db_mutex sqlite3_api->db_mutex -#define sqlite3_db_status sqlite3_api->db_status -#define sqlite3_extended_errcode sqlite3_api->extended_errcode -#define sqlite3_log sqlite3_api->log -#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 -#define sqlite3_sourceid sqlite3_api->sourceid -#define sqlite3_stmt_status sqlite3_api->stmt_status -#define sqlite3_strnicmp sqlite3_api->strnicmp -#define sqlite3_unlock_notify sqlite3_api->unlock_notify -#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint -#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint -#define sqlite3_wal_hook sqlite3_api->wal_hook -#endif /* SQLITE_CORE */ - -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; - -#endif /* _SQLITE3EXT_H_ */ - -/************** End of sqlite3ext.h ******************************************/ -/************** Continuing where we left off in loadext.c ********************/ - -#ifndef SQLITE_OMIT_LOAD_EXTENSION +static void hexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i, n; + const unsigned char *pBlob; + char *zHex, *z; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + pBlob = sqlite3_value_blob(argv[0]); + n = sqlite3_value_bytes(argv[0]); + assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + z = zHex = contextMalloc(context, ((i64)n)*2 + 1); + if( zHex ){ + for(i=0; i>4)&0xf]; + *(z++) = hexdigits[c&0xf]; + } + *z = 0; + sqlite3_result_text(context, zHex, n*2, sqlite3_free); + } +} /* -** Some API routines are omitted when various features are -** excluded from a build of SQLite. Substitute a NULL pointer -** for any missing APIs. +** The zeroblob(N) function returns a zero-filled blob of size N bytes. */ -#ifndef SQLITE_ENABLE_COLUMN_METADATA -# define sqlite3_column_database_name 0 -# define sqlite3_column_database_name16 0 -# define sqlite3_column_table_name 0 -# define sqlite3_column_table_name16 0 -# define sqlite3_column_origin_name 0 -# define sqlite3_column_origin_name16 0 -# define sqlite3_table_column_metadata 0 -#endif +static void zeroblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + i64 n; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( argc==1 ); + UNUSED_PARAMETER(argc); + n = sqlite3_value_int64(argv[0]); + testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + }else{ + sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + } +} -#ifdef SQLITE_OMIT_AUTHORIZATION -# define sqlite3_set_authorizer 0 -#endif +/* +** The replace() function. Three arguments are all strings: call +** them A, B, and C. The result is also a string which is derived +** from A by replacing every occurance of B with C. The match +** must be exact. Collating sequences are not used. +*/ +static void replaceFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zStr; /* The input string A */ + const unsigned char *zPattern; /* The pattern string B */ + const unsigned char *zRep; /* The replacement string C */ + unsigned char *zOut; /* The output */ + int nStr; /* Size of zStr */ + int nPattern; /* Size of zPattern */ + int nRep; /* Size of zRep */ + i64 nOut; /* Maximum size of zOut */ + int loopLimit; /* Last zStr[] that might match zPattern[] */ + int i, j; /* Loop counters */ -#ifdef SQLITE_OMIT_UTF16 -# define sqlite3_bind_text16 0 -# define sqlite3_collation_needed16 0 -# define sqlite3_column_decltype16 0 -# define sqlite3_column_name16 0 -# define sqlite3_column_text16 0 -# define sqlite3_complete16 0 -# define sqlite3_create_collation16 0 -# define sqlite3_create_function16 0 -# define sqlite3_errmsg16 0 -# define sqlite3_open16 0 -# define sqlite3_prepare16 0 -# define sqlite3_prepare16_v2 0 -# define sqlite3_result_error16 0 -# define sqlite3_result_text16 0 -# define sqlite3_result_text16be 0 -# define sqlite3_result_text16le 0 -# define sqlite3_value_text16 0 -# define sqlite3_value_text16be 0 -# define sqlite3_value_text16le 0 -# define sqlite3_column_database_name16 0 -# define sqlite3_column_table_name16 0 -# define sqlite3_column_origin_name16 0 -#endif + assert( argc==3 ); + UNUSED_PARAMETER(argc); + zStr = sqlite3_value_text(argv[0]); + if( zStr==0 ) return; + nStr = sqlite3_value_bytes(argv[0]); + assert( zStr==sqlite3_value_text(argv[0]) ); /* No encoding change */ + zPattern = sqlite3_value_text(argv[1]); + if( zPattern==0 ){ + assert( sqlite3_value_type(argv[1])==SQLITE_NULL + || sqlite3_context_db_handle(context)->mallocFailed ); + return; + } + if( zPattern[0]==0 ){ + assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); + sqlite3_result_value(context, argv[0]); + return; + } + nPattern = sqlite3_value_bytes(argv[1]); + assert( zPattern==sqlite3_value_text(argv[1]) ); /* No encoding change */ + zRep = sqlite3_value_text(argv[2]); + if( zRep==0 ) return; + nRep = sqlite3_value_bytes(argv[2]); + assert( zRep==sqlite3_value_text(argv[2]) ); + nOut = nStr + 1; + assert( nOutaLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + zOld = zOut; + zOut = sqlite3_realloc(zOut, (int)nOut); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + memcpy(&zOut[j], zRep, nRep); + j += nRep; + i += nPattern-1; + } + } + assert( j+nStr-i+1==nOut ); + memcpy(&zOut[j], &zStr[i], nStr-i); + j += nStr - i; + assert( j<=nOut ); + zOut[j] = 0; + sqlite3_result_text(context, (char*)zOut, j, sqlite3_free); +} -#ifdef SQLITE_OMIT_COMPLETE -# define sqlite3_complete 0 -# define sqlite3_complete16 0 -#endif +/* +** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. +** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. +*/ +static void trimFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn; /* Input string */ + const unsigned char *zCharSet; /* Set of characters to trim */ + int nIn; /* Number of bytes in input */ + int flags; /* 1: trimleft 2: trimright 3: trim */ + int i; /* Loop counter */ + unsigned char *aLen = 0; /* Length of each character in zCharSet */ + unsigned char **azChar = 0; /* Individual characters in zCharSet */ + int nChar; /* Number of characters in zCharSet */ -#ifdef SQLITE_OMIT_PROGRESS_CALLBACK -# define sqlite3_progress_handler 0 -#endif + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + return; + } + zIn = sqlite3_value_text(argv[0]); + if( zIn==0 ) return; + nIn = sqlite3_value_bytes(argv[0]); + assert( zIn==sqlite3_value_text(argv[0]) ); + if( argc==1 ){ + static const unsigned char lenOne[] = { 1 }; + static unsigned char * const azOne[] = { (u8*)" " }; + nChar = 1; + aLen = (u8*)lenOne; + azChar = (unsigned char **)azOne; + zCharSet = 0; + }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ + return; + }else{ + const unsigned char *z; + for(z=zCharSet, nChar=0; *z; nChar++){ + SQLITE_SKIP_UTF8(z); + } + if( nChar>0 ){ + azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+1)); + if( azChar==0 ){ + return; + } + aLen = (unsigned char*)&azChar[nChar]; + for(z=zCharSet, nChar=0; *z; nChar++){ + azChar[nChar] = (unsigned char *)z; + SQLITE_SKIP_UTF8(z); + aLen[nChar] = (u8)(z - azChar[nChar]); + } + } + } + if( nChar>0 ){ + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); + if( flags & 1 ){ + while( nIn>0 ){ + int len = 0; + for(i=0; i=nChar ) break; + zIn += len; + nIn -= len; + } + } + if( flags & 2 ){ + while( nIn>0 ){ + int len = 0; + for(i=0; i=nChar ) break; + nIn -= len; + } + } + if( zCharSet ){ + sqlite3_free(azChar); + } + } + sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); +} -#ifdef SQLITE_OMIT_VIRTUALTABLE -# define sqlite3_create_module 0 -# define sqlite3_create_module_v2 0 -# define sqlite3_declare_vtab 0 -#endif -#ifdef SQLITE_OMIT_SHARED_CACHE -# define sqlite3_enable_shared_cache 0 -#endif +/* IMP: R-25361-16150 This function is omitted from SQLite by default. It +** is only available if the SQLITE_SOUNDEX compile-time option is used +** when SQLite is built. +*/ +#ifdef SQLITE_SOUNDEX +/* +** Compute the soundex encoding of a word. +** +** IMP: R-59782-00072 The soundex(X) function returns a string that is the +** soundex encoding of the string X. +*/ +static void soundexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char zResult[8]; + const u8 *zIn; + int i, j; + static const unsigned char iCode[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + }; + assert( argc==1 ); + zIn = (u8*)sqlite3_value_text(argv[0]); + if( zIn==0 ) zIn = (u8*)""; + for(i=0; zIn[i] && !sqlite3Isalpha(zIn[i]); i++){} + if( zIn[i] ){ + u8 prevcode = iCode[zIn[i]&0x7f]; + zResult[0] = sqlite3Toupper(zIn[i]); + for(j=1; j<4 && zIn[i]; i++){ + int code = iCode[zIn[i]&0x7f]; + if( code>0 ){ + if( code!=prevcode ){ + prevcode = code; + zResult[j++] = code + '0'; + } + }else{ + prevcode = 0; + } + } + while( j<4 ){ + zResult[j++] = '0'; + } + zResult[j] = 0; + sqlite3_result_text(context, zResult, 4, SQLITE_TRANSIENT); + }else{ + /* IMP: R-64894-50321 The string "?000" is returned if the argument + ** is NULL or contains no ASCII alphabetic characters. */ + sqlite3_result_text(context, "?000", 4, SQLITE_STATIC); + } +} +#endif /* SQLITE_SOUNDEX */ -#ifdef SQLITE_OMIT_TRACE -# define sqlite3_profile 0 -# define sqlite3_trace 0 -#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** A function that loads a shared-library extension then returns NULL. +*/ +static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *zFile = (const char *)sqlite3_value_text(argv[0]); + const char *zProc; + sqlite3 *db = sqlite3_context_db_handle(context); + char *zErrMsg = 0; -#ifdef SQLITE_OMIT_GET_TABLE -# define sqlite3_free_table 0 -# define sqlite3_get_table 0 + if( argc==2 ){ + zProc = (const char *)sqlite3_value_text(argv[1]); + }else{ + zProc = 0; + } + if( zFile && sqlite3_load_extension(db, zFile, zProc, &zErrMsg) ){ + sqlite3_result_error(context, zErrMsg, -1); + sqlite3_free(zErrMsg); + } +} #endif -#ifdef SQLITE_OMIT_INCRBLOB -#define sqlite3_bind_zeroblob 0 -#define sqlite3_blob_bytes 0 -#define sqlite3_blob_close 0 -#define sqlite3_blob_open 0 -#define sqlite3_blob_read 0 -#define sqlite3_blob_write 0 -#endif /* -** The following structure contains pointers to all SQLite API routines. -** A pointer to this structure is passed into extensions when they are -** loaded so that the extension can make calls back into the SQLite -** library. -** -** When adding new APIs, add them to the bottom of this structure -** in order to preserve backwards compatibility. +** An instance of the following structure holds the context of a +** sum() or avg() aggregate computation. +*/ +typedef struct SumCtx SumCtx; +struct SumCtx { + double rSum; /* Floating point sum */ + i64 iSum; /* Integer sum */ + i64 cnt; /* Number of elements summed */ + u8 overflow; /* True if integer overflow seen */ + u8 approx; /* True if non-integer value was input to the sum */ +}; + +/* +** Routines used to compute the sum, average, and total. ** -** Extensions that use newer APIs should first call the -** sqlite3_libversion_number() to make sure that the API they -** intend to use is supported by the library. Extensions should -** also check to make sure that the pointer to the function is -** not NULL before calling it. +** The SUM() function follows the (broken) SQL standard which means +** that it returns NULL if it sums over no inputs. TOTAL returns +** 0.0 in that case. In addition, TOTAL always returns a float where +** SUM might return an integer if it never encounters a floating point +** value. TOTAL never fails, but SUM might through an exception if +** it overflows an integer. */ -static const sqlite3_api_routines sqlite3Apis = { - sqlite3_aggregate_context, -#ifndef SQLITE_OMIT_DEPRECATED - sqlite3_aggregate_count, -#else - 0, -#endif - sqlite3_bind_blob, - sqlite3_bind_double, - sqlite3_bind_int, - sqlite3_bind_int64, - sqlite3_bind_null, - sqlite3_bind_parameter_count, - sqlite3_bind_parameter_index, - sqlite3_bind_parameter_name, - sqlite3_bind_text, - sqlite3_bind_text16, - sqlite3_bind_value, - sqlite3_busy_handler, - sqlite3_busy_timeout, - sqlite3_changes, - sqlite3_close, - sqlite3_collation_needed, - sqlite3_collation_needed16, - sqlite3_column_blob, - sqlite3_column_bytes, - sqlite3_column_bytes16, - sqlite3_column_count, - sqlite3_column_database_name, - sqlite3_column_database_name16, - sqlite3_column_decltype, - sqlite3_column_decltype16, - sqlite3_column_double, - sqlite3_column_int, - sqlite3_column_int64, - sqlite3_column_name, - sqlite3_column_name16, - sqlite3_column_origin_name, - sqlite3_column_origin_name16, - sqlite3_column_table_name, - sqlite3_column_table_name16, - sqlite3_column_text, - sqlite3_column_text16, - sqlite3_column_type, - sqlite3_column_value, - sqlite3_commit_hook, - sqlite3_complete, - sqlite3_complete16, - sqlite3_create_collation, - sqlite3_create_collation16, - sqlite3_create_function, - sqlite3_create_function16, - sqlite3_create_module, - sqlite3_data_count, - sqlite3_db_handle, - sqlite3_declare_vtab, - sqlite3_enable_shared_cache, - sqlite3_errcode, - sqlite3_errmsg, - sqlite3_errmsg16, - sqlite3_exec, -#ifndef SQLITE_OMIT_DEPRECATED - sqlite3_expired, -#else - 0, -#endif - sqlite3_finalize, - sqlite3_free, - sqlite3_free_table, - sqlite3_get_autocommit, - sqlite3_get_auxdata, - sqlite3_get_table, - 0, /* Was sqlite3_global_recover(), but that function is deprecated */ - sqlite3_interrupt, - sqlite3_last_insert_rowid, - sqlite3_libversion, - sqlite3_libversion_number, - sqlite3_malloc, - sqlite3_mprintf, - sqlite3_open, - sqlite3_open16, - sqlite3_prepare, - sqlite3_prepare16, - sqlite3_profile, - sqlite3_progress_handler, - sqlite3_realloc, - sqlite3_reset, - sqlite3_result_blob, - sqlite3_result_double, - sqlite3_result_error, - sqlite3_result_error16, - sqlite3_result_int, - sqlite3_result_int64, - sqlite3_result_null, - sqlite3_result_text, - sqlite3_result_text16, - sqlite3_result_text16be, - sqlite3_result_text16le, - sqlite3_result_value, - sqlite3_rollback_hook, - sqlite3_set_authorizer, - sqlite3_set_auxdata, - sqlite3_snprintf, - sqlite3_step, - sqlite3_table_column_metadata, -#ifndef SQLITE_OMIT_DEPRECATED - sqlite3_thread_cleanup, -#else - 0, -#endif - sqlite3_total_changes, - sqlite3_trace, -#ifndef SQLITE_OMIT_DEPRECATED - sqlite3_transfer_bindings, -#else - 0, -#endif - sqlite3_update_hook, - sqlite3_user_data, - sqlite3_value_blob, - sqlite3_value_bytes, - sqlite3_value_bytes16, - sqlite3_value_double, - sqlite3_value_int, - sqlite3_value_int64, - sqlite3_value_numeric_type, - sqlite3_value_text, - sqlite3_value_text16, - sqlite3_value_text16be, - sqlite3_value_text16le, - sqlite3_value_type, - sqlite3_vmprintf, - /* - ** The original API set ends here. All extensions can call any - ** of the APIs above provided that the pointer is not NULL. But - ** before calling APIs that follow, extension should check the - ** sqlite3_libversion_number() to make sure they are dealing with - ** a library that is new enough to support that API. - ************************************************************************* - */ - sqlite3_overload_function, +static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + if( p && type!=SQLITE_NULL ){ + p->cnt++; + if( type==SQLITE_INTEGER ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum += v; + if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ + p->overflow = 1; + } + }else{ + p->rSum += sqlite3_value_double(argv[0]); + p->approx = 1; + } + } +} +static void sumFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + if( p->overflow ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( p->approx ){ + sqlite3_result_double(context, p->rSum); + }else{ + sqlite3_result_int64(context, p->iSum); + } + } +} +static void avgFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + sqlite3_result_double(context, p->rSum/(double)p->cnt); + } +} +static void totalFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + sqlite3_result_double(context, p ? p->rSum : (double)0); +} - /* - ** Added after 3.3.13 - */ - sqlite3_prepare_v2, - sqlite3_prepare16_v2, - sqlite3_clear_bindings, +/* +** The following structure keeps track of state information for the +** count() aggregate function. +*/ +typedef struct CountCtx CountCtx; +struct CountCtx { + i64 n; +}; - /* - ** Added for 3.4.1 - */ - sqlite3_create_module_v2, +/* +** Routines to implement the count() aggregate function. +*/ +static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){ + p->n++; + } - /* - ** Added for 3.5.0 - */ - sqlite3_bind_zeroblob, - sqlite3_blob_bytes, - sqlite3_blob_close, - sqlite3_blob_open, - sqlite3_blob_read, - sqlite3_blob_write, - sqlite3_create_collation_v2, - sqlite3_file_control, - sqlite3_memory_highwater, - sqlite3_memory_used, -#ifdef SQLITE_MUTEX_OMIT - 0, - 0, - 0, - 0, - 0, -#else - sqlite3_mutex_alloc, - sqlite3_mutex_enter, - sqlite3_mutex_free, - sqlite3_mutex_leave, - sqlite3_mutex_try, -#endif - sqlite3_open_v2, - sqlite3_release_memory, - sqlite3_result_error_nomem, - sqlite3_result_error_toobig, - sqlite3_sleep, - sqlite3_soft_heap_limit, - sqlite3_vfs_find, - sqlite3_vfs_register, - sqlite3_vfs_unregister, - - /* - ** Added for 3.5.8 - */ - sqlite3_threadsafe, - sqlite3_result_zeroblob, - sqlite3_result_error_code, - sqlite3_test_control, - sqlite3_randomness, - sqlite3_context_db_handle, - - /* - ** Added for 3.6.0 - */ - sqlite3_extended_result_codes, - sqlite3_limit, - sqlite3_next_stmt, - sqlite3_sql, - sqlite3_status, - - /* - ** Added for 3.7.4 - */ - sqlite3_backup_finish, - sqlite3_backup_init, - sqlite3_backup_pagecount, - sqlite3_backup_remaining, - sqlite3_backup_step, -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - sqlite3_compileoption_get, - sqlite3_compileoption_used, -#else - 0, - 0, -#endif - sqlite3_create_function_v2, - sqlite3_db_config, - sqlite3_db_mutex, - sqlite3_db_status, - sqlite3_extended_errcode, - sqlite3_log, - sqlite3_soft_heap_limit64, - sqlite3_sourceid, - sqlite3_stmt_status, - sqlite3_strnicmp, -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - sqlite3_unlock_notify, -#else - 0, -#endif -#ifndef SQLITE_OMIT_WAL - sqlite3_wal_autocheckpoint, - sqlite3_wal_checkpoint, - sqlite3_wal_hook, -#else - 0, - 0, - 0, +#ifndef SQLITE_OMIT_DEPRECATED + /* The sqlite3_aggregate_count() function is deprecated. But just to make + ** sure it still operates correctly, verify that its count agrees with our + ** internal count when using count(*) and when the total count can be + ** expressed as a 32-bit integer. */ + assert( argc==1 || p==0 || p->n>0x7fffffff + || p->n==sqlite3_aggregate_count(context) ); #endif -}; +} +static void countFinalize(sqlite3_context *context){ + CountCtx *p; + p = sqlite3_aggregate_context(context, 0); + sqlite3_result_int64(context, p ? p->n : 0); +} /* -** Attempt to load an SQLite extension library contained in the file -** zFile. The entry point is zProc. zProc may be 0 in which case a -** default entry point name (sqlite3_extension_init) is used. Use -** of the default name is recommended. -** -** Return SQLITE_OK on success and SQLITE_ERROR if something goes wrong. -** -** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with -** error message text. The calling function should free this memory -** by calling sqlite3DbFree(db, ). +** Routines to implement min() and max() aggregate functions. */ -static int sqlite3LoadExtension( - sqlite3 *db, /* Load the extension into this database connection */ - const char *zFile, /* Name of the shared library containing extension */ - const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ - char **pzErrMsg /* Put error message here if not 0 */ +static void minmaxStep( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv ){ - sqlite3_vfs *pVfs = db->pVfs; - void *handle; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); - char *zErrmsg = 0; - void **aHandle; - const int nMsg = 300; + Mem *pArg = (Mem *)argv[0]; + Mem *pBest; + UNUSED_PARAMETER(NotUsed); - if( pzErrMsg ) *pzErrMsg = 0; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); + if( !pBest ) return; - /* Ticket #1863. To avoid a creating security problems for older - ** applications that relink against newer versions of SQLite, the - ** ability to run load_extension is turned off by default. One - ** must call sqlite3_enable_load_extension() to turn on extension - ** loading. Otherwise you get the following error. - */ - if( (db->flags & SQLITE_LoadExtension)==0 ){ - if( pzErrMsg ){ - *pzErrMsg = sqlite3_mprintf("not authorized"); + if( pBest->flags ){ + int max; + int cmp; + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + /* This step function is used for both the min() and max() aggregates, + ** the only difference between the two being that the sense of the + ** comparison is inverted. For the max() aggregate, the + ** sqlite3_user_data() function returns (void *)-1. For min() it + ** returns (void *)db, where db is the sqlite3* database pointer. + ** Therefore the next statement sets variable 'max' to 1 for the max() + ** aggregate, or 0 for min(). + */ + max = sqlite3_user_data(context)!=0; + cmp = sqlite3MemCompare(pBest, pArg, pColl); + if( (max && cmp<0) || (!max && cmp>0) ){ + sqlite3VdbeMemCopy(pBest, pArg); } - return SQLITE_ERROR; + }else{ + sqlite3VdbeMemCopy(pBest, pArg); } - - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; +} +static void minMaxFinalize(sqlite3_context *context){ + sqlite3_value *pRes; + pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); + if( pRes ){ + if( ALWAYS(pRes->flags) ){ + sqlite3_result_value(context, pRes); + } + sqlite3VdbeMemRelease(pRes); } +} - handle = sqlite3OsDlOpen(pVfs, zFile); - if( handle==0 ){ - if( pzErrMsg ){ - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); - if( zErrmsg ){ - sqlite3_snprintf(nMsg, zErrmsg, - "unable to open shared library [%s]", zFile); - sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); +/* +** group_concat(EXPR, ?SEPARATOR?) +*/ +static void groupConcatStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zVal; + StrAccum *pAccum; + const char *zSep; + int nVal, nSep; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + + if( pAccum ){ + sqlite3 *db = sqlite3_context_db_handle(context); + int firstTerm = pAccum->useMalloc==0; + pAccum->useMalloc = 2; + pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; + if( !firstTerm ){ + if( argc==2 ){ + zSep = (char*)sqlite3_value_text(argv[1]); + nSep = sqlite3_value_bytes(argv[1]); + }else{ + zSep = ","; + nSep = 1; } + sqlite3StrAccumAppend(pAccum, zSep, nSep); } - return SQLITE_ERROR; + zVal = (char*)sqlite3_value_text(argv[0]); + nVal = sqlite3_value_bytes(argv[0]); + sqlite3StrAccumAppend(pAccum, zVal, nVal); } - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zProc); - if( xInit==0 ){ - if( pzErrMsg ){ - *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); - if( zErrmsg ){ - sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); - sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - } - sqlite3OsDlClose(pVfs, handle); - } - return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ - if( pzErrMsg ){ - *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); +} +static void groupConcatFinalize(sqlite3_context *context){ + StrAccum *pAccum; + pAccum = sqlite3_aggregate_context(context, 0); + if( pAccum ){ + if( pAccum->tooBig ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->mallocFailed ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, + sqlite3_free); } - sqlite3_free(zErrmsg); - sqlite3OsDlClose(pVfs, handle); - return SQLITE_ERROR; } +} - /* Append the new shared library handle to the db->aExtension array. */ - aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); - if( aHandle==0 ){ - return SQLITE_NOMEM; - } - if( db->nExtension>0 ){ - memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); +/* +** This routine does per-connection function registration. Most +** of the built-in functions above are part of the global function set. +** This routine only deals with those that are not global. +*/ +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ + int rc = sqlite3_overload_function(db, "MATCH", 2); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; } - sqlite3DbFree(db, db->aExtension); - db->aExtension = aHandle; - - db->aExtension[db->nExtension++] = handle; - return SQLITE_OK; -} -SQLITE_API int sqlite3_load_extension( - sqlite3 *db, /* Load the extension into this database connection */ - const char *zFile, /* Name of the shared library containing extension */ - const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ - char **pzErrMsg /* Put error message here if not 0 */ -){ - int rc; - sqlite3_mutex_enter(db->mutex); - rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; } /* -** Call this routine when the database connection is closing in order -** to clean up loaded extensions +** Set the LIKEOPT flag on the 2-argument function with the given name. */ -SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ - int i; - assert( sqlite3_mutex_held(db->mutex) ); - for(i=0; inExtension; i++){ - sqlite3OsDlClose(db->pVfs, db->aExtension[i]); +static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ + FuncDef *pDef; + pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), + 2, SQLITE_UTF8, 0); + if( ALWAYS(pDef) ){ + pDef->flags = flagVal; } - sqlite3DbFree(db, db->aExtension); } /* -** Enable or disable extension loading. Extension loading is disabled by -** default so as not to open security holes in older applications. +** Register the built-in LIKE and GLOB functions. The caseSensitive +** parameter determines whether or not the LIKE operator is case +** sensitive. GLOB is always case sensitive. */ -SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ - sqlite3_mutex_enter(db->mutex); - if( onoff ){ - db->flags |= SQLITE_LoadExtension; +SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + struct compareInfo *pInfo; + if( caseSensitive ){ + pInfo = (struct compareInfo*)&likeInfoAlt; }else{ - db->flags &= ~SQLITE_LoadExtension; + pInfo = (struct compareInfo*)&likeInfoNorm; } - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; + sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, + (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); + setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); + setLikeOptFlag(db, "like", + caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); } -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ - /* -** The auto-extension code added regardless of whether or not extension -** loading is supported. We need a dummy sqlite3Apis pointer for that -** code if regular extension loading is not available. This is that -** dummy pointer. +** pExpr points to an expression which implements a function. If +** it is appropriate to apply the LIKE optimization to that function +** then set aWc[0] through aWc[2] to the wildcard characters and +** return TRUE. If the function is not a LIKE-style function then +** return FALSE. */ -#ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; -#endif +SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ + FuncDef *pDef; + if( pExpr->op!=TK_FUNCTION + || !pExpr->x.pList + || pExpr->x.pList->nExpr!=2 + ){ + return 0; + } + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, + sqlite3Strlen30(pExpr->u.zToken), + 2, SQLITE_UTF8, 0); + if( NEVER(pDef==0) || (pDef->flags & SQLITE_FUNC_LIKE)==0 ){ + return 0; + } + /* The memcpy() statement assumes that the wildcard characters are + ** the first three statements in the compareInfo structure. The + ** asserts() that follow verify that assumption + */ + memcpy(aWc, pDef->pUserData, 3); + assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); + assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); + assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); + *pIsNocase = (pDef->flags & SQLITE_FUNC_CASE)==0; + return 1; +} /* -** The following object holds the list of automatically loaded -** extensions. +** All all of the FuncDef structures in the aBuiltinFunc[] array above +** to the global function hash table. This occurs at start-time (as +** a consequence of calling sqlite3_initialize()). ** -** This list is shared across threads. The SQLITE_MUTEX_STATIC_MASTER -** mutex must be held while accessing this list. -*/ -typedef struct sqlite3AutoExtList sqlite3AutoExtList; -static SQLITE_WSD struct sqlite3AutoExtList { - int nExt; /* Number of entries in aExt[] */ - void (**aExt)(void); /* Pointers to the extension init functions */ -} sqlite3Autoext = { 0, 0 }; - -/* The "wsdAutoext" macro will resolve to the autoextension -** state vector. If writable static data is unsupported on the target, -** we have to locate the state vector at run-time. In the more common -** case where writable static data is supported, wsdStat can refer directly -** to the "sqlite3Autoext" state vector declared above. +** After this routine runs */ -#ifdef SQLITE_OMIT_WSD -# define wsdAutoextInit \ - sqlite3AutoExtList *x = &GLOBAL(sqlite3AutoExtList,sqlite3Autoext) -# define wsdAutoext x[0] -#else -# define wsdAutoextInit -# define wsdAutoext sqlite3Autoext +SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ + /* + ** The following array holds FuncDef structures for all of the functions + ** defined in this file. + ** + ** The array cannot be constant since changes are made to the + ** FuncDef.pHash elements at start-time. The elements of this array + ** are read-only after initialization is complete. + */ + static SQLITE_WSD FuncDef aBuiltinFunc[] = { + FUNCTION(ltrim, 1, 1, 0, trimFunc ), + FUNCTION(ltrim, 2, 1, 0, trimFunc ), + FUNCTION(rtrim, 1, 2, 0, trimFunc ), + FUNCTION(rtrim, 2, 2, 0, trimFunc ), + FUNCTION(trim, 1, 3, 0, trimFunc ), + FUNCTION(trim, 2, 3, 0, trimFunc ), + FUNCTION(min, -1, 0, 1, minmaxFunc ), + FUNCTION(min, 0, 0, 1, 0 ), + AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + FUNCTION(max, -1, 1, 1, minmaxFunc ), + FUNCTION(max, 0, 1, 1, 0 ), + AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + FUNCTION(typeof, 1, 0, 0, typeofFunc ), + FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(abs, 1, 0, 0, absFunc ), +#ifndef SQLITE_OMIT_FLOATING_POINT + FUNCTION(round, 1, 0, 0, roundFunc ), + FUNCTION(round, 2, 0, 0, roundFunc ), #endif + FUNCTION(upper, 1, 0, 0, upperFunc ), + FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), +/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ + {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION(hex, 1, 0, 0, hexFunc ), +/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ + {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, + FUNCTION(random, 0, 0, 0, randomFunc ), + FUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION(nullif, 2, 0, 1, nullifFunc ), + FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + FUNCTION(quote, 1, 0, 0, quoteFunc ), + FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + FUNCTION(changes, 0, 0, 0, changes ), + FUNCTION(total_changes, 0, 0, 0, total_changes ), + FUNCTION(replace, 3, 0, 0, replaceFunc ), + FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), + #ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), + #endif + #ifndef SQLITE_OMIT_LOAD_EXTENSION + FUNCTION(load_extension, 1, 0, 0, loadExt ), + FUNCTION(load_extension, 2, 0, 0, loadExt ), + #endif + AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), + AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), + AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), + /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ + {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, + AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), + AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), + AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), + + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + #ifdef SQLITE_CASE_SENSITIVE_LIKE + LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + #else + LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), + LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), + #endif + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); -/* -** Register a statically linked extension that is automatically -** loaded by every new database connection. -*/ -SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ - int rc = SQLITE_OK; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ){ - return rc; - }else -#endif - { - int i; -#if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); -#endif - wsdAutoextInit; - sqlite3_mutex_enter(mutex); - for(i=0; i=wsdAutoext.nExt ){ - xInit = 0; - go = 0; - }else{ - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - wsdAutoext.aExt[i]; - } - sqlite3_mutex_leave(mutex); - zErrmsg = 0; - if( xInit && xInit(db, &zErrmsg, &sqlite3Apis) ){ - sqlite3Error(db, SQLITE_ERROR, - "automatic extension loading failed: %s", zErrmsg); - go = 0; - } - sqlite3_free(zErrmsg); - } -} - -/************** End of loadext.c *********************************************/ -/************** Begin file pragma.c ******************************************/ -/* -** 2003 April 6 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -85325,1506 +83822,1201 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains code used to implement the PRAGMA command. +** This file contains code used by the compiler to add foreign key +** support to compiled SQL statements. */ -/* Ignore this whole file if pragmas are disabled -*/ -#if !defined(SQLITE_OMIT_PRAGMA) +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER /* -** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or -** unrecognized string argument. +** Deferred and Immediate FKs +** -------------------------- ** -** Note that the values returned are one less that the values that -** should be passed into sqlite3BtreeSetSafetyLevel(). The is done -** to support legacy SQL code. The safety level used to be boolean -** and older scripts may have used numbers 0 for OFF and 1 for ON. -*/ -static u8 getSafetyLevel(const char *z){ - /* 123456789 123456789 */ - static const char zText[] = "onoffalseyestruefull"; - static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; - static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; - static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; - int i, n; - if( sqlite3Isdigit(*z) ){ - return (u8)sqlite3Atoi(z); - } - n = sqlite3Strlen30(z); - for(i=0; idb->mallocFailed flag is set. */ -static int getAutoVacuum(const char *z){ - int i; - if( 0==sqlite3StrICmp(z, "none") ) return BTREE_AUTOVACUUM_NONE; - if( 0==sqlite3StrICmp(z, "full") ) return BTREE_AUTOVACUUM_FULL; - if( 0==sqlite3StrICmp(z, "incremental") ) return BTREE_AUTOVACUUM_INCR; - i = sqlite3Atoi(z); - return (u8)((i>=0&&i<=2)?i:0); -} -#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ +static int locateFkeyIndex( + Parse *pParse, /* Parse context to store any error in */ + Table *pParent, /* Parent table of FK constraint pFKey */ + FKey *pFKey, /* Foreign key to find index for */ + Index **ppIdx, /* OUT: Unique index on parent table */ + int **paiCol /* OUT: Map of index columns in pFKey */ +){ + Index *pIdx = 0; /* Value to return via *ppIdx */ + int *aiCol = 0; /* Value to return via *paiCol */ + int nCol = pFKey->nCol; /* Number of columns in parent key */ + char *zKey = pFKey->aCol[0].zCol; /* Name of left-most parent key column */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* -** Interpret the given string as a temp db location. Return 1 for file -** backed temporary databases, 2 for the Red-Black tree in memory database -** and 0 to use the compile-time default. -*/ -static int getTempStore(const char *z){ - if( z[0]>='0' && z[0]<='2' ){ - return z[0] - '0'; - }else if( sqlite3StrICmp(z, "file")==0 ){ - return 1; - }else if( sqlite3StrICmp(z, "memory")==0 ){ - return 2; - }else{ - return 0; - } -} -#endif /* SQLITE_PAGER_PRAGMAS */ + /* The caller is responsible for zeroing output parameters. */ + assert( ppIdx && *ppIdx==0 ); + assert( !paiCol || *paiCol==0 ); + assert( pParse ); -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* -** Invalidate temp storage, either when the temp storage is changed -** from default, or when 'file' and the temp_store_directory has changed -*/ -static int invalidateTempStorage(Parse *pParse){ - sqlite3 *db = pParse->db; - if( db->aDb[1].pBt!=0 ){ - if( !db->autoCommit || sqlite3BtreeIsInReadTrans(db->aDb[1].pBt) ){ - sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " - "from within a transaction"); - return SQLITE_ERROR; + /* If this is a non-composite (single column) foreign key, check if it + ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx + ** and *paiCol set to zero and return early. + ** + ** Otherwise, for a composite foreign key (more than one column), allocate + ** space for the aiCol array (returned via output parameter *paiCol). + ** Non-composite foreign keys do not require the aiCol array. + */ + if( nCol==1 ){ + /* The FK maps to the IPK if any of the following are true: + ** + ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly + ** mapped to the primary key of table pParent, or + ** 2) The FK is explicitly mapped to a column declared as INTEGER + ** PRIMARY KEY. + */ + if( pParent->iPKey>=0 ){ + if( !zKey ) return 0; + if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zName, zKey) ) return 0; } - sqlite3BtreeClose(db->aDb[1].pBt); - db->aDb[1].pBt = 0; - sqlite3ResetInternalSchema(db, 0); - } - return SQLITE_OK; -} -#endif /* SQLITE_PAGER_PRAGMAS */ - -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* -** If the TEMP database is open, close it and mark the database schema -** as needing reloading. This must be done when using the SQLITE_TEMP_STORE -** or DEFAULT_TEMP_STORE pragmas. -*/ -static int changeTempStorage(Parse *pParse, const char *zStorageType){ - int ts = getTempStore(zStorageType); - sqlite3 *db = pParse->db; - if( db->temp_store==ts ) return SQLITE_OK; - if( invalidateTempStorage( pParse ) != SQLITE_OK ){ - return SQLITE_ERROR; - } - db->temp_store = (u8)ts; - return SQLITE_OK; -} -#endif /* SQLITE_PAGER_PRAGMAS */ - -/* -** Generate code to return a single integer value. -*/ -static void returnSingleInt(Parse *pParse, const char *zLabel, i64 value){ - Vdbe *v = sqlite3GetVdbe(pParse); - int mem = ++pParse->nMem; - i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); - if( pI64 ){ - memcpy(pI64, &value, sizeof(value)); + }else if( paiCol ){ + assert( nCol>1 ); + aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); + if( !aiCol ) return 1; + *paiCol = aiCol; } - sqlite3VdbeAddOp4(v, OP_Int64, 0, mem, 0, (char*)pI64, P4_INT64); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); -} - -#ifndef SQLITE_OMIT_FLAG_PRAGMAS -/* -** Check to see if zRight and zLeft refer to a pragma that queries -** or changes one of the flags in db->flags. Return 1 if so and 0 if not. -** Also, implement the pragma. -*/ -static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ - static const struct sPragmaType { - const char *zName; /* Name of the pragma */ - int mask; /* Mask for the db->flags value */ - } aPragma[] = { - { "full_column_names", SQLITE_FullColNames }, - { "short_column_names", SQLITE_ShortColNames }, - { "count_changes", SQLITE_CountRows }, - { "empty_result_callbacks", SQLITE_NullCallback }, - { "legacy_file_format", SQLITE_LegacyFileFmt }, - { "fullfsync", SQLITE_FullFSync }, - { "checkpoint_fullfsync", SQLITE_CkptFullFSync }, - { "reverse_unordered_selects", SQLITE_ReverseOrder }, -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX - { "automatic_index", SQLITE_AutoIndex }, -#endif -#ifdef SQLITE_DEBUG - { "sql_trace", SQLITE_SqlTrace }, - { "vdbe_listing", SQLITE_VdbeListing }, - { "vdbe_trace", SQLITE_VdbeTrace }, -#endif -#ifndef SQLITE_OMIT_CHECK - { "ignore_check_constraints", SQLITE_IgnoreChecks }, -#endif - /* The following is VERY experimental */ - { "writable_schema", SQLITE_WriteSchema|SQLITE_RecoveryMode }, - { "omit_readlock", SQLITE_NoReadlock }, - /* TODO: Maybe it shouldn't be possible to change the ReadUncommitted - ** flag if there are any active statements. */ - { "read_uncommitted", SQLITE_ReadUncommitted }, - { "recursive_triggers", SQLITE_RecTriggers }, + for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->nColumn==nCol && pIdx->onError!=OE_None ){ + /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number + ** of columns. If each indexed column corresponds to a foreign key + ** column of pFKey, then this index is a winner. */ - /* This flag may only be set if both foreign-key and trigger support - ** are present in the build. */ -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { "foreign_keys", SQLITE_ForeignKeys }, -#endif - }; - int i; - const struct sPragmaType *p; - for(i=0, p=aPragma; izName)==0 ){ - sqlite3 *db = pParse->db; - Vdbe *v; - v = sqlite3GetVdbe(pParse); - assert( v!=0 ); /* Already allocated by sqlite3Pragma() */ - if( ALWAYS(v) ){ - if( zRight==0 ){ - returnSingleInt(pParse, p->zName, (db->flags & p->mask)!=0 ); - }else{ - int mask = p->mask; /* Mask of bits to set or clear. */ - if( db->autoCommit==0 ){ - /* Foreign key support may not be enabled or disabled while not - ** in auto-commit mode. */ - mask &= ~(SQLITE_ForeignKeys); + if( zKey==0 ){ + /* If zKey is NULL, then this foreign key is implicitly mapped to + ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be + ** identified by the test (Index.autoIndex==2). */ + if( pIdx->autoIndex==2 ){ + if( aiCol ){ + int i; + for(i=0; iaCol[i].iFrom; } + break; + } + }else{ + /* If zKey is non-NULL, then this foreign key was declared to + ** map to an explicit list of columns in table pParent. Check if this + ** index matches those columns. Also, check that the index uses + ** the default collation sequences for each column. */ + int i, j; + for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ + char *zDfltColl; /* Def. collation for column */ + char *zIdxCol; /* Name of indexed column */ - if( getBoolean(zRight) ){ - db->flags |= mask; - }else{ - db->flags &= ~mask; + /* If the index uses a collation sequence that is different from + ** the default collation sequence for the column, this index is + ** unusable. Bail out early in this case. */ + zDfltColl = pParent->aCol[iCol].zColl; + if( !zDfltColl ){ + zDfltColl = "BINARY"; } + if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; - /* Many of the flag-pragmas modify the code generated by the SQL - ** compiler (eg. count_changes). So add an opcode to expire all - ** compiled SQL statements after modifying a pragma value. - */ - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + zIdxCol = pParent->aCol[iCol].zName; + for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ + if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; + break; + } + } + if( j==nCol ) break; } + if( i==nCol ) break; /* pIdx is usable */ } + } + } - return 1; + if( !pIdx ){ + if( !pParse->disableTriggers ){ + sqlite3ErrorMsg(pParse, "foreign key mismatch"); } + sqlite3DbFree(pParse->db, aiCol); + return 1; } + + *ppIdx = pIdx; return 0; } -#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ /* -** Return a human-readable name for a constraint resolution action. -*/ -#ifndef SQLITE_OMIT_FOREIGN_KEY -static const char *actionName(u8 action){ - const char *zName; - switch( action ){ - case OE_SetNull: zName = "SET NULL"; break; - case OE_SetDflt: zName = "SET DEFAULT"; break; - case OE_Cascade: zName = "CASCADE"; break; - case OE_Restrict: zName = "RESTRICT"; break; - default: zName = "NO ACTION"; - assert( action==OE_None ); break; +** This function is called when a row is inserted into or deleted from the +** child table of foreign key constraint pFKey. If an SQL UPDATE is executed +** on the child table of pFKey, this function is invoked twice for each row +** affected - once to "delete" the old row, and then again to "insert" the +** new row. +** +** Each time it is called, this function generates VDBE code to locate the +** row in the parent table that corresponds to the row being inserted into +** or deleted from the child table. If the parent row can be found, no +** special action is taken. Otherwise, if the parent row can *not* be +** found in the parent table: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** INSERT immediate Increment the "immediate constraint counter". +** +** DELETE immediate Decrement the "immediate constraint counter". +** +** INSERT deferred Increment the "deferred constraint counter". +** +** DELETE deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.1" and "D.1". +*/ +static void fkLookupParent( + Parse *pParse, /* Parse context */ + int iDb, /* Index of database housing pTab */ + Table *pTab, /* Parent table of FK pFKey */ + Index *pIdx, /* Unique index on parent key columns in pTab */ + FKey *pFKey, /* Foreign key constraint */ + int *aiCol, /* Map from parent key columns to child table columns */ + int regData, /* Address of array containing child table row */ + int nIncr, /* Increment constraint counter by this */ + int isIgnore /* If true, pretend pTab contains all NULL values */ +){ + int i; /* Iterator variable */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ + int iCur = pParse->nTab - 1; /* Cursor number to use */ + int iOk = sqlite3VdbeMakeLabel(v); /* jump here if parent key found */ + + /* If nIncr is less than zero, then check at runtime if there are any + ** outstanding constraints to resolve. If there are not, there is no need + ** to check if deleting this row resolves any outstanding violations. + ** + ** Check if any of the key columns in the child table row are NULL. If + ** any are, then the constraint is considered satisfied. No need to + ** search for a matching row in the parent table. */ + if( nIncr<0 ){ + sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); + } + for(i=0; inCol; i++){ + int iReg = aiCol[i] + regData + 1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); } - return zName; -} -#endif + if( isIgnore==0 ){ + if( pIdx==0 ){ + /* If pIdx is NULL, then the parent key is the INTEGER PRIMARY KEY + ** column of the parent table (table pTab). */ + int iMustBeInt; /* Address of MustBeInt instruction */ + int regTemp = sqlite3GetTempReg(pParse); + + /* Invoke MustBeInt to coerce the child key value to an integer (i.e. + ** apply the affinity of the parent key). If this fails, then there + ** is no matching parent key. Before using MustBeInt, make a copy of + ** the value. Otherwise, the value inserted into the child key column + ** will have INTEGER affinity applied to it, which may not be correct. */ + sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); + iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); + + /* If the parent table is the same as the child table, and we are about + ** to increment the constraint-counter (i.e. this is an INSERT operation), + ** then check if the row being inserted matches itself. If so, do not + ** increment the constraint-counter. */ + if( pTab==pFKey->pFrom && nIncr==1 ){ + sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); + } + + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + sqlite3VdbeJumpHere(v, iMustBeInt); + sqlite3ReleaseTempReg(pParse, regTemp); + }else{ + int nCol = pFKey->nCol; + int regTemp = sqlite3GetTempRange(pParse, nCol); + int regRec = sqlite3GetTempReg(pParse); + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + + sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + for(i=0; ipFrom && nIncr==1 ){ + int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; + for(i=0; iaiColumn[i]+1+regData; + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } + sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + } + + sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec); + sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); + + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempRange(pParse, regTemp, nCol); + } + } -/* -** Parameter eMode must be one of the PAGER_JOURNALMODE_XXX constants -** defined in pager.h. This function returns the associated lowercase -** journal-mode name. -*/ -SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ - static char * const azModeName[] = { - "delete", "persist", "off", "truncate", "memory" -#ifndef SQLITE_OMIT_WAL - , "wal" -#endif - }; - assert( PAGER_JOURNALMODE_DELETE==0 ); - assert( PAGER_JOURNALMODE_PERSIST==1 ); - assert( PAGER_JOURNALMODE_OFF==2 ); - assert( PAGER_JOURNALMODE_TRUNCATE==3 ); - assert( PAGER_JOURNALMODE_MEMORY==4 ); - assert( PAGER_JOURNALMODE_WAL==5 ); - assert( eMode>=0 && eMode<=ArraySize(azModeName) ); + if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){ + /* Special case: If this is an INSERT statement that will insert exactly + ** one row into the table, raise a constraint immediately instead of + ** incrementing a counter. This is necessary as the VM code is being + ** generated for will not open a statement transaction. */ + assert( nIncr==1 ); + sqlite3HaltConstraint( + pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + ); + }else{ + if( nIncr>0 && pFKey->isDeferred==0 ){ + sqlite3ParseToplevel(pParse)->mayAbort = 1; + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + } - if( eMode==ArraySize(azModeName) ) return 0; - return azModeName[eMode]; + sqlite3VdbeResolveLabel(v, iOk); + sqlite3VdbeAddOp1(v, OP_Close, iCur); } /* -** Process a pragma statement. +** This function is called to generate code executed when a row is deleted +** from the parent table of foreign key constraint pFKey and, if pFKey is +** deferred, when a row is inserted into the same table. When generating +** code for an SQL UPDATE operation, this function may be called twice - +** once to "delete" the old row and once to "insert" the new row. ** -** Pragmas are of this form: +** The code generated by this function scans through the rows in the child +** table that correspond to the parent table row being deleted or inserted. +** For each child row found, one of the following actions is taken: ** -** PRAGMA [database.]id [= value] +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** DELETE immediate Increment the "immediate constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "foreign key constraint failed" exception. ** -** The identifier might also be a string. The value is a string, and -** identifier, or a number. If minusFlag is true, then the value is -** a number that was preceded by a minus sign. +** INSERT immediate Decrement the "immediate constraint counter". ** -** If the left side is "database.id" then pId1 is the database name -** and pId2 is the id. If the left side is just "id" then pId1 is the -** id and pId2 is any empty string. +** DELETE deferred Increment the "deferred constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "foreign key constraint failed" exception. +** +** INSERT deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.2" and "D.2". */ -SQLITE_PRIVATE void sqlite3Pragma( - Parse *pParse, - Token *pId1, /* First part of [database.]id field */ - Token *pId2, /* Second part of [database.]id field, or NULL */ - Token *pValue, /* Token for , or NULL */ - int minusFlag /* True if a '-' sign preceded */ +static void fkScanChildren( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* SrcList containing the table to scan */ + Table *pTab, + Index *pIdx, /* Foreign key index */ + FKey *pFKey, /* Foreign key relationship */ + int *aiCol, /* Map from pIdx cols to child table cols */ + int regData, /* Referenced table data starts here */ + int nIncr /* Amount to increment deferred counter by */ ){ - char *zLeft = 0; /* Nul-terminated UTF-8 string */ - char *zRight = 0; /* Nul-terminated UTF-8 string , or NULL */ - const char *zDb = 0; /* The database name */ - Token *pId; /* Pointer to token */ - int iDb; /* Database index for */ - sqlite3 *db = pParse->db; - Db *pDb; - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); - if( v==0 ) return; - sqlite3VdbeRunOnlyOnce(v); - pParse->nMem = 2; - - /* Interpret the [database.] part of the pragma statement. iDb is the - ** index of the database this pragma is being applied to in db.aDb[]. */ - iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); - if( iDb<0 ) return; - pDb = &db->aDb[iDb]; + sqlite3 *db = pParse->db; /* Database handle */ + int i; /* Iterator variable */ + Expr *pWhere = 0; /* WHERE clause to scan with */ + NameContext sNameContext; /* Context used to resolve WHERE clause */ + WhereInfo *pWInfo; /* Context used by sqlite3WhereXXX() */ + int iFkIfZero = 0; /* Address of OP_FkIfZero */ + Vdbe *v = sqlite3GetVdbe(pParse); - /* If the temp database has been explicitly named as part of the - ** pragma, make sure it is open. - */ - if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ - return; - } + assert( !pIdx || pIdx->pTable==pTab ); - zLeft = sqlite3NameFromToken(db, pId); - if( !zLeft ) return; - if( minusFlag ){ - zRight = sqlite3MPrintf(db, "-%T", pValue); - }else{ - zRight = sqlite3NameFromToken(db, pValue); + if( nIncr<0 ){ + iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); } - assert( pId2 ); - zDb = pId2->n>0 ? pDb->zName : 0; - if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ - goto pragma_out; - } - -#ifndef SQLITE_OMIT_PAGER_PRAGMAS - /* - ** PRAGMA [database.]default_cache_size - ** PRAGMA [database.]default_cache_size=N + /* Create an Expr object representing an SQL expression like: ** - ** The first form reports the current persistent setting for the - ** page cache size. The value returned is the maximum number of - ** pages in the page cache. The second form sets both the current - ** page cache size value and the persistent page cache size value - ** stored in the database file. + ** = AND = ... ** - ** Older versions of SQLite would set the default cache size to a - ** negative number to indicate synchronous=OFF. These days, synchronous - ** is always on by default regardless of the sign of the default cache - ** size. But continue to take the absolute value of the default cache - ** size of historical compatibility. + ** The collation sequence used for the comparison should be that of + ** the parent key columns. The affinity of the parent key column should + ** be applied to each child key value before the comparison takes place. */ - if( sqlite3StrICmp(zLeft,"default_cache_size")==0 ){ - static const VdbeOpList getCacheSize[] = { - { OP_Transaction, 0, 0, 0}, /* 0 */ - { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, - { OP_Integer, 0, 2, 0}, - { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, - { OP_Integer, 0, 1, 0}, /* 6 */ - { OP_ResultRow, 1, 1, 0}, - }; - int addr; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeUsesBtree(v, iDb); - if( !zRight ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); - pParse->nMem += 2; - addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); - }else{ - int size = sqlite3Atoi(zRight); - if( size<0 ) size = -size; - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, size, 1); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); - pDb->pSchema->cache_size = size; - sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); - } - }else + for(i=0; inCol; i++){ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + Expr *pEq; /* Expression (pLeft = pRight) */ + int iCol; /* Index of column in child table */ + const char *zCol; /* Name of column in child table */ - /* - ** PRAGMA [database.]page_size - ** PRAGMA [database.]page_size=N - ** - ** The first form reports the current setting for the - ** database page size in bytes. The second form sets the - ** database page size value. The value can only be set if - ** the database has not yet been created. - */ - if( sqlite3StrICmp(zLeft,"page_size")==0 ){ - Btree *pBt = pDb->pBt; - assert( pBt!=0 ); - if( !zRight ){ - int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(pParse, "page_size", size); - }else{ - /* Malloc may fail when setting the page-size, as there is an internal - ** buffer that the pager module resizes using sqlite3_realloc(). - */ - db->nextPagesize = sqlite3Atoi(zRight); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ - db->mallocFailed = 1; + pLeft = sqlite3Expr(db, TK_REGISTER, 0); + if( pLeft ){ + /* Set the collation sequence and affinity of the LHS of each TK_EQ + ** expression to the parent key column defaults. */ + if( pIdx ){ + Column *pCol; + iCol = pIdx->aiColumn[i]; + pCol = &pTab->aCol[iCol]; + if( pTab->iPKey==iCol ) iCol = -1; + pLeft->iTable = regData+iCol+1; + pLeft->affinity = pCol->affinity; + pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); + }else{ + pLeft->iTable = regData; + pLeft->affinity = SQLITE_AFF_INTEGER; } } - }else + iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iCol>=0 ); + zCol = pFKey->pFrom->aCol[iCol].zName; + pRight = sqlite3Expr(db, TK_ID, zCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); + pWhere = sqlite3ExprAnd(db, pWhere, pEq); + } - /* - ** PRAGMA [database.]secure_delete - ** PRAGMA [database.]secure_delete=ON/OFF - ** - ** The first form reports the current setting for the - ** secure_delete flag. The second form changes the secure_delete - ** flag setting and reports thenew value. - */ - if( sqlite3StrICmp(zLeft,"secure_delete")==0 ){ - Btree *pBt = pDb->pBt; - int b = -1; - assert( pBt!=0 ); - if( zRight ){ - b = getBoolean(zRight); - } - if( pId2->n==0 && b>=0 ){ - int ii; - for(ii=0; iinDb; ii++){ - sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); - } + /* If the child table is the same as the parent table, and this scan + ** is taking place as part of a DELETE operation (operation D.2), omit the + ** row being deleted from the scan by adding ($rowid != rowid) to the WHERE + ** clause, where $rowid is the rowid of the row being deleted. */ + if( pTab==pFKey->pFrom && nIncr>0 ){ + Expr *pEq; /* Expression (pLeft = pRight) */ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + pLeft = sqlite3Expr(db, TK_REGISTER, 0); + pRight = sqlite3Expr(db, TK_COLUMN, 0); + if( pLeft && pRight ){ + pLeft->iTable = regData; + pLeft->affinity = SQLITE_AFF_INTEGER; + pRight->iTable = pSrc->a[0].iCursor; + pRight->iColumn = -1; } - b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(pParse, "secure_delete", b); - }else + pEq = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); + pWhere = sqlite3ExprAnd(db, pWhere, pEq); + } - /* - ** PRAGMA [database.]max_page_count - ** PRAGMA [database.]max_page_count=N - ** - ** The first form reports the current setting for the - ** maximum number of pages in the database file. The - ** second form attempts to change this setting. Both - ** forms return the current setting. - ** - ** PRAGMA [database.]page_count - ** - ** Return the number of pages in the specified database. - */ - if( sqlite3StrICmp(zLeft,"page_count")==0 - || sqlite3StrICmp(zLeft,"max_page_count")==0 - ){ - int iReg; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3CodeVerifySchema(pParse, iDb); - iReg = ++pParse->nMem; - if( zLeft[0]=='p' ){ - sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); - }else{ - sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); - } - sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); - }else + /* Resolve the references in the WHERE clause. */ + memset(&sNameContext, 0, sizeof(NameContext)); + sNameContext.pSrcList = pSrc; + sNameContext.pParse = pParse; + sqlite3ResolveExprNames(&sNameContext, pWhere); - /* - ** PRAGMA [database.]locking_mode - ** PRAGMA [database.]locking_mode = (normal|exclusive) - */ - if( sqlite3StrICmp(zLeft,"locking_mode")==0 ){ - const char *zRet = "normal"; - int eMode = getLockingMode(zRight); + /* Create VDBE to loop through the entries in pSrc that match the WHERE + ** clause. If the constraint is not deferred, throw an exception for + ** each row found. Otherwise, for deferred constraints, increment the + ** deferred constraint counter by nIncr for each row selected. */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); + if( nIncr>0 && pFKey->isDeferred==0 ){ + sqlite3ParseToplevel(pParse)->mayAbort = 1; + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + if( pWInfo ){ + sqlite3WhereEnd(pWInfo); + } - if( pId2->n==0 && eMode==PAGER_LOCKINGMODE_QUERY ){ - /* Simple "PRAGMA locking_mode;" statement. This is a query for - ** the current default locking mode (which may be different to - ** the locking-mode of the main database). - */ - eMode = db->dfltLockMode; - }else{ - Pager *pPager; - if( pId2->n==0 ){ - /* This indicates that no database name was specified as part - ** of the PRAGMA command. In this case the locking-mode must be - ** set on all attached databases, as well as the main db file. - ** - ** Also, the sqlite3.dfltLockMode variable is set so that - ** any subsequently attached databases also use the specified - ** locking mode. - */ - int ii; - assert(pDb==&db->aDb[0]); - for(ii=2; iinDb; ii++){ - pPager = sqlite3BtreePager(db->aDb[ii].pBt); - sqlite3PagerLockingMode(pPager, eMode); - } - db->dfltLockMode = (u8)eMode; - } - pPager = sqlite3BtreePager(pDb->pBt); - eMode = sqlite3PagerLockingMode(pPager, eMode); - } + /* Clean up the WHERE clause constructed above. */ + sqlite3ExprDelete(db, pWhere); + if( iFkIfZero ){ + sqlite3VdbeJumpHere(v, iFkIfZero); + } +} - assert(eMode==PAGER_LOCKINGMODE_NORMAL||eMode==PAGER_LOCKINGMODE_EXCLUSIVE); - if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ - zRet = "exclusive"; - } - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - }else +/* +** This function returns a pointer to the head of a linked list of FK +** constraints for which table pTab is the parent table. For example, +** given the following schema: +** +** CREATE TABLE t1(a PRIMARY KEY); +** CREATE TABLE t2(b REFERENCES t1(a); +** +** Calling this function with table "t1" as an argument returns a pointer +** to the FKey structure representing the foreign key constraint on table +** "t2". Calling this function with "t2" as the argument would return a +** NULL pointer (as there are no FK constraints for which t2 is the parent +** table). +*/ +SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ + int nName = sqlite3Strlen30(pTab->zName); + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); +} - /* - ** PRAGMA [database.]journal_mode - ** PRAGMA [database.]journal_mode = - ** (delete|persist|off|truncate|memory|wal|off) - */ - if( sqlite3StrICmp(zLeft,"journal_mode")==0 ){ - int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ - int ii; /* Loop counter */ +/* +** The second argument is a Trigger structure allocated by the +** fkActionTrigger() routine. This function deletes the Trigger structure +** and all of its sub-components. +** +** The Trigger structure or any of its sub-components may be allocated from +** the lookaside buffer belonging to database handle dbMem. +*/ +static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ + if( p ){ + TriggerStep *pStep = p->step_list; + sqlite3ExprDelete(dbMem, pStep->pWhere); + sqlite3ExprListDelete(dbMem, pStep->pExprList); + sqlite3SelectDelete(dbMem, pStep->pSelect); + sqlite3ExprDelete(dbMem, p->pWhen); + sqlite3DbFree(dbMem, p); + } +} - /* Force the schema to be loaded on all databases. This cases all - ** database files to be opened and the journal_modes set. */ - if( sqlite3ReadSchema(pParse) ){ - goto pragma_out; +/* +** This function is called to generate code that runs when table pTab is +** being dropped from the database. The SrcList passed as the second argument +** to this function contains a single entry guaranteed to resolve to +** table pTab. +** +** Normally, no code is required. However, if either +** +** (a) The table is the parent table of a FK constraint, or +** (b) The table is the child table of a deferred FK constraint and it is +** determined at runtime that there are outstanding deferred FK +** constraint violations in the database, +** +** then the equivalent of "DELETE FROM " is executed before dropping +** the table from the database. Triggers are disabled while running this +** DELETE, but foreign key actions are not. +*/ +SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ + sqlite3 *db = pParse->db; + if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) && !pTab->pSelect ){ + int iSkip = 0; + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( v ); /* VDBE has already been allocated */ + if( sqlite3FkReferences(pTab)==0 ){ + /* Search for a deferred foreign key constraint for which this table + ** is the child table. If one cannot be found, return without + ** generating any VDBE code. If one can be found, then jump over + ** the entire DELETE if there are no outstanding deferred constraints + ** when this statement is run. */ + FKey *p; + for(p=pTab->pFKey; p; p=p->pNextFrom){ + if( p->isDeferred ) break; + } + if( !p ) return; + iSkip = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); } - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); + pParse->disableTriggers = 1; + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0); + pParse->disableTriggers = 0; - if( zRight==0 ){ - /* If there is no "=MODE" part of the pragma, do a query for the - ** current mode */ - eMode = PAGER_JOURNALMODE_QUERY; + /* If the DELETE has generated immediate foreign key constraint + ** violations, halt the VDBE and return an error at this point, before + ** any modifications to the schema are made. This is because statement + ** transactions are not able to rollback schema changes. */ + sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3HaltConstraint( + pParse, OE_Abort, "foreign key constraint failed", P4_STATIC + ); + + if( iSkip ){ + sqlite3VdbeResolveLabel(v, iSkip); + } + } +} + +/* +** This function is called when inserting, deleting or updating a row of +** table pTab to generate VDBE code to perform foreign key constraint +** processing for the operation. +** +** For a DELETE operation, parameter regOld is passed the index of the +** first register in an array of (pTab->nCol+1) registers containing the +** rowid of the row being deleted, followed by each of the column values +** of the row being deleted, from left to right. Parameter regNew is passed +** zero in this case. +** +** For an INSERT operation, regOld is passed zero and regNew is passed the +** first register of an array of (pTab->nCol+1) registers containing the new +** row data. +** +** For an UPDATE operation, this function is called twice. Once before +** the original record is deleted from the table using the calling convention +** described for DELETE. Then again after the original record is deleted +** but before the new record is inserted using the INSERT convention. +*/ +SQLITE_PRIVATE void sqlite3FkCheck( + Parse *pParse, /* Parse context */ + Table *pTab, /* Row is being deleted from this table */ + int regOld, /* Previous row data is stored here */ + int regNew /* New row data is stored here */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + FKey *pFKey; /* Used to iterate through FKs */ + int iDb; /* Index of database containing pTab */ + const char *zDb; /* Name of database containing pTab */ + int isIgnoreErrors = pParse->disableTriggers; + + /* Exactly one of regOld and regNew should be non-zero. */ + assert( (regOld==0)!=(regNew==0) ); + + /* If foreign-keys are disabled, this function is a no-op. */ + if( (db->flags&SQLITE_ForeignKeys)==0 ) return; + + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zName; + + /* Loop through all the foreign key constraints for which pTab is the + ** child table (the table that the foreign key definition is part of). */ + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + Table *pTo; /* Parent table of foreign key pFKey */ + Index *pIdx = 0; /* Index on key columns in pTo */ + int *aiFree = 0; + int *aiCol; + int iCol; + int i; + int isIgnore = 0; + + /* Find the parent table of this foreign key. Also find a unique index + ** on the parent key columns in the parent table. If either of these + ** schema items cannot be located, set an error in pParse and return + ** early. */ + if( pParse->disableTriggers ){ + pTo = sqlite3FindTable(db, pFKey->zTo, zDb); }else{ - const char *zMode; - int n = sqlite3Strlen30(zRight); - for(eMode=0; (zMode = sqlite3JournalModename(eMode))!=0; eMode++){ - if( sqlite3StrNICmp(zRight, zMode, n)==0 ) break; - } - if( !zMode ){ - /* If the "=MODE" part does not match any known journal mode, - ** then do a query */ - eMode = PAGER_JOURNALMODE_QUERY; - } + pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); } - if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ - /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ - iDb = 0; - pId2->n = 1; + if( !pTo || locateFkeyIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + if( !isIgnoreErrors || db->mallocFailed ) return; + continue; } - for(ii=db->nDb-1; ii>=0; ii--){ - if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ - sqlite3VdbeUsesBtree(v, ii); - sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode); + assert( pFKey->nCol==1 || (aiFree && pIdx) ); + + if( aiFree ){ + aiCol = aiFree; + }else{ + iCol = pFKey->aCol[0].iFrom; + aiCol = &iCol; + } + for(i=0; inCol; i++){ + if( aiCol[i]==pTab->iPKey ){ + aiCol[i] = -1; } +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Request permission to read the parent key columns. If the + ** authorization callback returns SQLITE_IGNORE, behave as if any + ** values read from the parent table are NULL. */ + if( db->xAuth ){ + int rcauth; + char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; + rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); + isIgnore = (rcauth==SQLITE_IGNORE); + } +#endif } - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - }else - /* - ** PRAGMA [database.]journal_size_limit - ** PRAGMA [database.]journal_size_limit=N - ** - ** Get or set the size limit on rollback journal files. - */ - if( sqlite3StrICmp(zLeft,"journal_size_limit")==0 ){ - Pager *pPager = sqlite3BtreePager(pDb->pBt); - i64 iLimit = -2; - if( zRight ){ - sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); - if( iLimit<-1 ) iLimit = -1; + /* Take a shared-cache advisory read-lock on the parent table. Allocate + ** a cursor to use to search the unique index on the parent key columns + ** in the parent table. */ + sqlite3TableLock(pParse, iDb, pTo->tnum, 0, pTo->zName); + pParse->nTab++; + + if( regOld!=0 ){ + /* A row is being removed from the child table. Search for the parent. + ** If the parent does not exist, removing the child row resolves an + ** outstanding foreign key constraint violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); + } + if( regNew!=0 ){ + /* A row is being added to the child table. If a parent row cannot + ** be found, adding the child row has violated the FK constraint. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); } - iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(pParse, "journal_size_limit", iLimit); - }else -#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + sqlite3DbFree(db, aiFree); + } - /* - ** PRAGMA [database.]auto_vacuum - ** PRAGMA [database.]auto_vacuum=N - ** - ** Get or set the value of the database 'auto-vacuum' parameter. - ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL - */ -#ifndef SQLITE_OMIT_AUTOVACUUM - if( sqlite3StrICmp(zLeft,"auto_vacuum")==0 ){ - Btree *pBt = pDb->pBt; - assert( pBt!=0 ); - if( sqlite3ReadSchema(pParse) ){ - goto pragma_out; + /* Loop through all the foreign key constraints that refer to this table */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + Index *pIdx = 0; /* Foreign key index for pFKey */ + SrcList *pSrc; + int *aiCol = 0; + + if( !pFKey->isDeferred && !pParse->pToplevel && !pParse->isMultiWrite ){ + assert( regOld==0 && regNew!=0 ); + /* Inserting a single row into a parent table cannot cause an immediate + ** foreign key violation. So do nothing in this case. */ + continue; } - if( !zRight ){ - int auto_vacuum; - if( ALWAYS(pBt) ){ - auto_vacuum = sqlite3BtreeGetAutoVacuum(pBt); - }else{ - auto_vacuum = SQLITE_DEFAULT_AUTOVACUUM; + + if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( !isIgnoreErrors || db->mallocFailed ) return; + continue; + } + assert( aiCol || pFKey->nCol==1 ); + + /* Create a SrcList structure containing a single table (the table + ** the foreign key that refers to this table is attached to). This + ** is required for the sqlite3WhereXXX() interface. */ + pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + if( pSrc ){ + struct SrcList_item *pItem = pSrc->a; + pItem->pTab = pFKey->pFrom; + pItem->zName = pFKey->pFrom->zName; + pItem->pTab->nRef++; + pItem->iCursor = pParse->nTab++; + + if( regNew!=0 ){ + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); } - returnSingleInt(pParse, "auto_vacuum", auto_vacuum); - }else{ - int eAuto = getAutoVacuum(zRight); - assert( eAuto>=0 && eAuto<=2 ); - db->nextAutovac = (u8)eAuto; - if( ALWAYS(eAuto>=0) ){ - /* Call SetAutoVacuum() to set initialize the internal auto and - ** incr-vacuum flags. This is required in case this connection - ** creates the database file. It is important that it is created - ** as an auto-vacuum capable db. - */ - int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); - if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ - /* When setting the auto_vacuum mode to either "full" or - ** "incremental", write the value of meta[6] in the database - ** file. Before writing to meta[6], check that meta[3] indicates - ** that this really is an auto-vacuum capable database. - */ - static const VdbeOpList setMeta6[] = { - { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, - { OP_If, 1, 0, 0}, /* 2 */ - { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ - { OP_Integer, 0, 1, 0}, /* 4 */ - { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ - }; - int iAddr; - iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6); - sqlite3VdbeChangeP1(v, iAddr, iDb); - sqlite3VdbeChangeP1(v, iAddr+1, iDb); - sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); - sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); - sqlite3VdbeChangeP1(v, iAddr+5, iDb); - sqlite3VdbeUsesBtree(v, iDb); - } + if( regOld!=0 ){ + /* If there is a RESTRICT action configured for the current operation + ** on the parent table of this FK, then throw an exception + ** immediately if the FK constraint is violated, even if this is a + ** deferred trigger. That's what RESTRICT means. To defer checking + ** the constraint, the FK should specify NO ACTION (represented + ** using OE_None). NO ACTION is the default. */ + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); } + pItem->zName = 0; + sqlite3SrcListDelete(db, pSrc); } - }else -#endif + sqlite3DbFree(db, aiCol); + } +} - /* - ** PRAGMA [database.]incremental_vacuum(N) - ** - ** Do N steps of incremental vacuuming on a database. - */ -#ifndef SQLITE_OMIT_AUTOVACUUM - if( sqlite3StrICmp(zLeft,"incremental_vacuum")==0 ){ - int iLimit, addr; - if( sqlite3ReadSchema(pParse) ){ - goto pragma_out; - } - if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ - iLimit = 0x7fffffff; - } - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); - addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); - sqlite3VdbeAddOp1(v, OP_ResultRow, 1); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); - sqlite3VdbeJumpHere(v, addr); - }else -#endif +#define COLUMN_MASK(x) (((x)>31) ? 0xffffffff : ((u32)1<<(x))) -#ifndef SQLITE_OMIT_PAGER_PRAGMAS - /* - ** PRAGMA [database.]cache_size - ** PRAGMA [database.]cache_size=N - ** - ** The first form reports the current local setting for the - ** page cache size. The local setting can be different from - ** the persistent cache size value that is stored in the database - ** file itself. The value returned is the maximum number of - ** pages in the page cache. The second form sets the local - ** page cache size value. It does not change the persistent - ** cache size stored on the disk so the cache size will revert - ** to its default value when the database is closed and reopened. - ** N should be a positive integer. - */ - if( sqlite3StrICmp(zLeft,"cache_size")==0 ){ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - if( !zRight ){ - returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); - }else{ - int size = sqlite3Atoi(zRight); - if( size<0 ) size = -size; - pDb->pSchema->cache_size = size; - sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. +*/ +SQLITE_PRIVATE u32 sqlite3FkOldmask( + Parse *pParse, /* Parse context */ + Table *pTab /* Table being modified */ +){ + u32 mask = 0; + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *p; + int i; + for(p=pTab->pFKey; p; p=p->pNextFrom){ + for(i=0; inCol; i++) mask |= COLUMN_MASK(p->aCol[i].iFrom); } - }else - - /* - ** PRAGMA temp_store - ** PRAGMA temp_store = "default"|"memory"|"file" - ** - ** Return or set the local value of the temp_store flag. Changing - ** the local value does not make changes to the disk file and the default - ** value will be restored the next time the database is opened. - ** - ** Note that it is possible for the library compile-time options to - ** override this setting - */ - if( sqlite3StrICmp(zLeft, "temp_store")==0 ){ - if( !zRight ){ - returnSingleInt(pParse, "temp_store", db->temp_store); - }else{ - changeTempStorage(pParse, zRight); + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + Index *pIdx = 0; + locateFkeyIndex(pParse, pTab, p, &pIdx, 0); + if( pIdx ){ + for(i=0; inColumn; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } } - }else + } + return mask; +} - /* - ** PRAGMA temp_store_directory - ** PRAGMA temp_store_directory = ""|"directory_name" - ** - ** Return or set the local value of the temp_store_directory flag. Changing - ** the value sets a specific directory to be used for temporary files. - ** Setting to a null string reverts to the default temporary directory search. - ** If temporary directory is changed, then invalidateTempStorage. - ** - */ - if( sqlite3StrICmp(zLeft, "temp_store_directory")==0 ){ - if( !zRight ){ - if( sqlite3_temp_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "temp_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. If the operation is a DELETE, then +** parameter aChange is passed a NULL value. For an UPDATE, aChange points +** to an array of size N, where N is the number of columns in table pTab. +** If the i'th column is not modified by the UPDATE, then the corresponding +** entry in the aChange[] array is set to -1. If the column is modified, +** the value is 0 or greater. Parameter chngRowid is set to true if the +** UPDATE statement modifies the rowid fields of the table. +** +** If any foreign key processing will be required, this function returns +** true. If there is no foreign key related processing, this function +** returns false. +*/ +SQLITE_PRIVATE int sqlite3FkRequired( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being modified */ + int *aChange, /* Non-NULL for UPDATE operations */ + int chngRowid /* True for UPDATE that affects rowid */ +){ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + if( !aChange ){ + /* A DELETE operation. Foreign key processing is required if the + ** table in question is either the child or parent table for any + ** foreign key constraint. */ + return (sqlite3FkReferences(pTab) || pTab->pFKey); }else{ -#ifndef SQLITE_OMIT_WSD - if( zRight[0] ){ - int rc; - int res; - rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); - if( rc!=SQLITE_OK || res==0 ){ - sqlite3ErrorMsg(pParse, "not a writable directory"); - goto pragma_out; + /* This is an UPDATE. Foreign key processing is only required if the + ** operation modifies one or more child or parent key columns. */ + int i; + FKey *p; + + /* Check if any child key columns are being modified. */ + for(p=pTab->pFKey; p; p=p->pNextFrom){ + for(i=0; inCol; i++){ + int iChildKey = p->aCol[i].iFrom; + if( aChange[iChildKey]>=0 ) return 1; + if( iChildKey==pTab->iPKey && chngRowid ) return 1; } } - if( SQLITE_TEMP_STORE==0 - || (SQLITE_TEMP_STORE==1 && db->temp_store<=1) - || (SQLITE_TEMP_STORE==2 && db->temp_store==1) - ){ - invalidateTempStorage(pParse); - } - sqlite3_free(sqlite3_temp_directory); - if( zRight[0] ){ - sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); - }else{ - sqlite3_temp_directory = 0; + + /* Check if any parent key columns are being modified. */ + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + for(i=0; inCol; i++){ + char *zKey = p->aCol[i].zCol; + int iKey; + for(iKey=0; iKeynCol; iKey++){ + Column *pCol = &pTab->aCol[iKey]; + if( (zKey ? !sqlite3StrICmp(pCol->zName, zKey) : pCol->isPrimKey) ){ + if( aChange[iKey]>=0 ) return 1; + if( iKey==pTab->iPKey && chngRowid ) return 1; + } + } + } } -#endif /* SQLITE_OMIT_WSD */ } - }else + } + return 0; +} -#if !defined(SQLITE_ENABLE_LOCKING_STYLE) -# if defined(__APPLE__) -# define SQLITE_ENABLE_LOCKING_STYLE 1 -# else -# define SQLITE_ENABLE_LOCKING_STYLE 0 -# endif -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - /* - ** PRAGMA [database.]lock_proxy_file - ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" - ** - ** Return or set the value of the lock_proxy_file flag. Changing - ** the value sets a specific file to be used for database access locks. - ** - */ - if( sqlite3StrICmp(zLeft, "lock_proxy_file")==0 ){ - if( !zRight ){ - Pager *pPager = sqlite3BtreePager(pDb->pBt); - char *proxy_file_path = NULL; - sqlite3_file *pFile = sqlite3PagerFile(pPager); - sqlite3OsFileControl(pFile, SQLITE_GET_LOCKPROXYFILE, - &proxy_file_path); - - if( proxy_file_path ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "lock_proxy_file", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } - }else{ - Pager *pPager = sqlite3BtreePager(pDb->pBt); - sqlite3_file *pFile = sqlite3PagerFile(pPager); - int res; - if( zRight[0] ){ - res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, - zRight); - } else { - res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, - NULL); - } - if( res!=SQLITE_OK ){ - sqlite3ErrorMsg(pParse, "failed to set lock proxy file"); - goto pragma_out; - } - } - }else -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - - /* - ** PRAGMA [database.]synchronous - ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL - ** - ** Return or set the local value of the synchronous flag. Changing - ** the local value does not make changes to the disk file and the - ** default value will be restored the next time the database is - ** opened. - */ - if( sqlite3StrICmp(zLeft,"synchronous")==0 ){ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - if( !zRight ){ - returnSingleInt(pParse, "synchronous", pDb->safety_level-1); - }else{ - if( !db->autoCommit ){ - sqlite3ErrorMsg(pParse, - "Safety level may not be changed inside a transaction"); - }else{ - pDb->safety_level = getSafetyLevel(zRight)+1; - } - } - }else -#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ +/* +** This function is called when an UPDATE or DELETE operation is being +** compiled on table pTab, which is the parent table of foreign-key pFKey. +** If the current operation is an UPDATE, then the pChanges parameter is +** passed a pointer to the list of columns being modified. If it is a +** DELETE, pChanges is passed a NULL pointer. +** +** It returns a pointer to a Trigger structure containing a trigger +** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. +** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is +** returned (these actions require no special handling by the triggers +** sub-system, code for them is created by fkScanChildren()). +** +** For example, if pFKey is the foreign key and pTab is table "p" in +** the following schema: +** +** CREATE TABLE p(pk PRIMARY KEY); +** CREATE TABLE c(ck REFERENCES p ON DELETE CASCADE); +** +** then the returned trigger structure is equivalent to: +** +** CREATE TRIGGER ... DELETE ON p BEGIN +** DELETE FROM c WHERE ck = old.pk; +** END; +** +** The returned pointer is cached as part of the foreign key object. It +** is eventually freed along with the rest of the foreign key object by +** sqlite3FkDelete(). +*/ +static Trigger *fkActionTrigger( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + FKey *pFKey, /* Foreign key to get action for */ + ExprList *pChanges /* Change-list for UPDATE, NULL for DELETE */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int action; /* One of OE_None, OE_Cascade etc. */ + Trigger *pTrigger; /* Trigger definition to return */ + int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ -#ifndef SQLITE_OMIT_FLAG_PRAGMAS - if( flagPragma(pParse, zLeft, zRight) ){ - /* The flagPragma() subroutine also generates any necessary code - ** there is nothing more to do here */ - }else -#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ + action = pFKey->aAction[iAction]; + pTrigger = pFKey->apTrigger[iAction]; -#ifndef SQLITE_OMIT_SCHEMA_PRAGMAS - /* - ** PRAGMA table_info(
    ) - ** - ** Return a single row for each column of the named table. The columns of - ** the returned data set are: - ** - ** cid: Column id (numbered from left to right, starting at 0) - ** name: Column name - ** type: Column declaration type. - ** notnull: True if 'NOT NULL' is part of column declaration - ** dflt_value: The default value for the column, if any. - */ - if( sqlite3StrICmp(zLeft, "table_info")==0 && zRight ){ - Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - pTab = sqlite3FindTable(db, zRight, zDb); - if( pTab ){ - int i; - int nHidden = 0; - Column *pCol; - sqlite3VdbeSetNumCols(v, 6); - pParse->nMem = 6; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); - sqlite3ViewGetColumnNames(pParse, pTab); - for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ - if( IsHiddenColumn(pCol) ){ - nHidden++; - continue; - } - sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - pCol->zType ? pCol->zType : "", 0); - sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); - if( pCol->zDflt ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); + if( action!=OE_None && !pTrigger ){ + u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ + char const *zFrom; /* Name of child table */ + int nFrom; /* Length in bytes of zFrom */ + Index *pIdx = 0; /* Parent key index for this FK */ + int *aiCol = 0; /* child table cols -> parent key cols */ + TriggerStep *pStep = 0; /* First (only) step of trigger program */ + Expr *pWhere = 0; /* WHERE clause of trigger step */ + ExprList *pList = 0; /* Changes list if ON UPDATE CASCADE */ + Select *pSelect = 0; /* If RESTRICT, "SELECT RAISE(...)" */ + int i; /* Iterator variable */ + Expr *pWhen = 0; /* WHEN clause for the trigger */ + + if( locateFkeyIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + assert( aiCol || pFKey->nCol==1 ); + + for(i=0; inCol; i++){ + Token tOld = { "old", 3 }; /* Literal "old" token */ + Token tNew = { "new", 3 }; /* Literal "new" token */ + Token tFromCol; /* Name of column in child table */ + Token tToCol; /* Name of column in parent table */ + int iFromCol; /* Idx of column in child table */ + Expr *pEq; /* tFromCol = OLD.tToCol */ + + iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iFromCol>=0 ); + tToCol.z = pIdx ? pTab->aCol[pIdx->aiColumn[i]].zName : "oid"; + tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; + + tToCol.n = sqlite3Strlen30(tToCol.z); + tFromCol.n = sqlite3Strlen30(tFromCol.z); + + /* Create the expression "OLD.zToCol = zFromCol". It is important + ** that the "OLD.zToCol" term is on the LHS of the = operator, so + ** that the affinity and collation sequence associated with the + ** parent table are used for the comparison. */ + pEq = sqlite3PExpr(pParse, TK_EQ, + sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + , 0), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tFromCol) + , 0); + pWhere = sqlite3ExprAnd(db, pWhere, pEq); + + /* For ON UPDATE, construct the next term of the WHEN clause. + ** The final WHEN clause will be like this: + ** + ** WHEN NOT(old.col1 IS new.col1 AND ... AND old.colN IS new.colN) + */ + if( pChanges ){ + pEq = sqlite3PExpr(pParse, TK_IS, + sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tOld), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + 0), + sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol), + 0), + 0); + pWhen = sqlite3ExprAnd(db, pWhen, pEq); + } + + if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ + Expr *pNew; + if( action==OE_Cascade ){ + pNew = sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_ID, 0, 0, &tNew), + sqlite3PExpr(pParse, TK_ID, 0, 0, &tToCol) + , 0); + }else if( action==OE_SetDflt ){ + Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; + if( pDflt ){ + pNew = sqlite3ExprDup(db, pDflt, 0); + }else{ + pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + } }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, 5); + pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); } - sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); + pList = sqlite3ExprListAppend(pParse, pList, pNew); + sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); } } - }else + sqlite3DbFree(db, aiCol); - if( sqlite3StrICmp(zLeft, "index_info")==0 && zRight ){ - Index *pIdx; - Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - pIdx = sqlite3FindIndex(db, zRight, zDb); - if( pIdx ){ - int i; - pTab = pIdx->pTable; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); - for(i=0; inColumn; i++){ - int cnum = pIdx->aiColumn[i]; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); - assert( pTab->nCol>cnum ); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + zFrom = pFKey->pFrom->zName; + nFrom = sqlite3Strlen30(zFrom); + + if( action==OE_Restrict ){ + Token tFrom; + Expr *pRaise; + + tFrom.z = zFrom; + tFrom.n = nFrom; + pRaise = sqlite3Expr(db, TK_RAISE, "foreign key constraint failed"); + if( pRaise ){ + pRaise->affinity = OE_Abort; } + pSelect = sqlite3SelectNew(pParse, + sqlite3ExprListAppend(pParse, 0, pRaise), + sqlite3SrcListAppend(db, 0, &tFrom, 0), + pWhere, + 0, 0, 0, 0, 0, 0 + ); + pWhere = 0; } - }else - if( sqlite3StrICmp(zLeft, "index_list")==0 && zRight ){ - Index *pIdx; - Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - pTab = sqlite3FindTable(db, zRight, zDb); - if( pTab ){ - v = sqlite3GetVdbe(pParse); - pIdx = pTab->pIndex; - if( pIdx ){ - int i = 0; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); - while(pIdx){ - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); - ++i; - pIdx = pIdx->pNext; - } + /* Disable lookaside memory allocation */ + enableLookaside = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; + + pTrigger = (Trigger *)sqlite3DbMallocZero(db, + sizeof(Trigger) + /* struct Trigger */ + sizeof(TriggerStep) + /* Single step in trigger program */ + nFrom + 1 /* Space for pStep->target.z */ + ); + if( pTrigger ){ + pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; + pStep->target.z = (char *)&pStep[1]; + pStep->target.n = nFrom; + memcpy((char *)pStep->target.z, zFrom, nFrom); + + pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); + pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( pWhen ){ + pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0, 0); + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); } } - }else - if( sqlite3StrICmp(zLeft, "database_list")==0 ){ - int i; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); - for(i=0; inDb; i++){ - if( db->aDb[i].pBt==0 ) continue; - assert( db->aDb[i].zName!=0 ); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); - } - }else + /* Re-enable the lookaside buffer, if it was disabled earlier. */ + db->lookaside.bEnabled = enableLookaside; - if( sqlite3StrICmp(zLeft, "collation_list")==0 ){ - int i = 0; - HashElem *p; - sqlite3VdbeSetNumCols(v, 2); - pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ - CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); + sqlite3ExprDelete(db, pWhere); + sqlite3ExprDelete(db, pWhen); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed==1 ){ + fkTriggerDelete(db, pTrigger); + return 0; } - }else -#endif /* SQLITE_OMIT_SCHEMA_PRAGMAS */ -#ifndef SQLITE_OMIT_FOREIGN_KEY - if( sqlite3StrICmp(zLeft, "foreign_key_list")==0 && zRight ){ - FKey *pFK; - Table *pTab; - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - pTab = sqlite3FindTable(db, zRight, zDb); - if( pTab ){ - v = sqlite3GetVdbe(pParse); - pFK = pTab->pFKey; - if( pFK ){ - int i = 0; - sqlite3VdbeSetNumCols(v, 8); - pParse->nMem = 8; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); - while(pFK){ - int j; - for(j=0; jnCol; j++){ - char *zCol = pFK->aCol[j].zCol; - char *zOnDelete = (char *)actionName(pFK->aAction[0]); - char *zOnUpdate = (char *)actionName(pFK->aAction[1]); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, j, 2); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pTab->aCol[pFK->aCol[j].iFrom].zName, 0); - sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); - } - ++i; - pFK = pFK->pNextFrom; + switch( action ){ + case OE_Restrict: + pStep->op = TK_SELECT; + break; + case OE_Cascade: + if( !pChanges ){ + pStep->op = TK_DELETE; + break; } + default: + pStep->op = TK_UPDATE; + } + pStep->pTrig = pTrigger; + pTrigger->pSchema = pTab->pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pFKey->apTrigger[iAction] = pTrigger; + pTrigger->op = (pChanges ? TK_UPDATE : TK_DELETE); + } + + return pTrigger; +} + +/* +** This function is called when deleting or updating a row to implement +** any required CASCADE, SET NULL or SET DEFAULT actions. +*/ +SQLITE_PRIVATE void sqlite3FkActions( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + ExprList *pChanges, /* Change-list for UPDATE, NULL for DELETE */ + int regOld /* Address of array containing old row */ +){ + /* If foreign-key support is enabled, iterate through all FKs that + ** refer to table pTab. If there is an action associated with the FK + ** for this operation (either update or delete), invoke the associated + ** trigger sub-program. */ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *pFKey; /* Iterator variable */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + Trigger *pAction = fkActionTrigger(pParse, pTab, pFKey, pChanges); + if( pAction ){ + sqlite3CodeRowTriggerDirect(pParse, pAction, pTab, regOld, OE_Abort, 0); } } - }else -#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + } +} -#ifndef NDEBUG - if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ - if( zRight ){ - if( getBoolean(zRight) ){ - sqlite3ParserTrace(stderr, "parser: "); +#endif /* ifndef SQLITE_OMIT_TRIGGER */ + +/* +** Free all memory associated with foreign key definitions attached to +** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash +** hash table. +*/ +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ + FKey *pFKey; /* Iterator variable */ + FKey *pNext; /* Copy of pFKey->pNextFrom */ + + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ + + /* Remove the FK from the fkeyHash hash table. */ + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ - sqlite3ParserTrace(0, 0); + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; } } - }else -#endif - /* Reinstall the LIKE and GLOB functions. The variant of LIKE - ** used will be case sensitive or not depending on the RHS. - */ - if( sqlite3StrICmp(zLeft, "case_sensitive_like")==0 ){ - if( zRight ){ - sqlite3RegisterLikeFunctions(db, getBoolean(zRight)); - } - }else + /* EV: R-30323-21917 Each foreign key constraint in SQLite is + ** classified as either immediate or deferred. + */ + assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); -#ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX -# define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); #endif -#ifndef SQLITE_OMIT_INTEGRITY_CHECK - /* Pragma "quick_check" is an experimental reduced version of - ** integrity_check designed to detect most database corruption - ** without most of the overhead of a full integrity-check. - */ - if( sqlite3StrICmp(zLeft, "integrity_check")==0 - || sqlite3StrICmp(zLeft, "quick_check")==0 - ){ - int i, j, addr, mxErr; - - /* Code that appears at the end of the integrity check. If no error - ** messages have been generated, output OK. Otherwise output the - ** error message - */ - static const VdbeOpList endCode[] = { - { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_IfNeg, 1, 0, 0}, /* 1 */ - { OP_String8, 0, 3, 0}, /* 2 */ - { OP_ResultRow, 3, 1, 0}, - }; - - int isQuick = (zLeft[0]=='q'); - - /* Initialize the VDBE program */ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - pParse->nMem = 6; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); - - /* Set the maximum error count */ - mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; - if( zRight ){ - sqlite3GetInt32(zRight, &mxErr); - if( mxErr<=0 ){ - mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; - } - } - sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ - - /* Do an integrity check on each database file */ - for(i=0; inDb; i++){ - HashElem *x; - Hash *pTbls; - int cnt = 0; - - if( OMIT_TEMPDB && i==1 ) continue; - - sqlite3CodeVerifySchema(pParse, i); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); - - /* Do an integrity check of the B-Tree - ** - ** Begin by filling registers 2, 3, ... with the root pages numbers - ** for all tables and indices in the database. - */ - pTbls = &db->aDb[i].pSchema->tblHash; - for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ - Table *pTab = sqliteHashData(x); - Index *pIdx; - sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); - cnt++; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); - cnt++; - } - } - - /* Make sure sufficient number of registers have been allocated */ - if( pParse->nMem < cnt+4 ){ - pParse->nMem = cnt+4; - } - - /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); - sqlite3VdbeChangeP5(v, (u8)i); - addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), - P4_DYNAMIC); - sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); - sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); - sqlite3VdbeJumpHere(v, addr); - - /* Make sure all the indices are constructed correctly. - */ - for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ - Table *pTab = sqliteHashData(x); - Index *pIdx; - int loopTop; - - if( pTab->pIndex==0 ) continue; - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); - sqlite3OpenTableAndIndices(pParse, pTab, 1, OP_OpenRead); - sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */ - loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0); - sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2; - int r1; - static const VdbeOpList idxErr[] = { - { OP_AddImm, 1, -1, 0}, - { OP_String8, 0, 3, 0}, /* 1 */ - { OP_Rowid, 1, 4, 0}, - { OP_String8, 0, 5, 0}, /* 3 */ - { OP_String8, 0, 6, 0}, /* 4 */ - { OP_Concat, 4, 3, 3}, - { OP_Concat, 5, 3, 3}, - { OP_Concat, 6, 3, 3}, - { OP_ResultRow, 3, 1, 0}, - { OP_IfPos, 1, 0, 0}, /* 9 */ - { OP_Halt, 0, 0, 0}, - }; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0); - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1); - addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr); - sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC); - sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC); - sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_STATIC); - sqlite3VdbeJumpHere(v, addr+9); - sqlite3VdbeJumpHere(v, jmp2); - } - sqlite3VdbeAddOp2(v, OP_Next, 1, loopTop+1); - sqlite3VdbeJumpHere(v, loopTop); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - static const VdbeOpList cntIdx[] = { - { OP_Integer, 0, 3, 0}, - { OP_Rewind, 0, 0, 0}, /* 1 */ - { OP_AddImm, 3, 1, 0}, - { OP_Next, 0, 0, 0}, /* 3 */ - { OP_Eq, 2, 0, 3}, /* 4 */ - { OP_AddImm, 1, -1, 0}, - { OP_String8, 0, 2, 0}, /* 6 */ - { OP_String8, 0, 3, 0}, /* 7 */ - { OP_Concat, 3, 2, 2}, - { OP_ResultRow, 2, 1, 0}, - }; - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); - addr = sqlite3VdbeAddOpList(v, ArraySize(cntIdx), cntIdx); - sqlite3VdbeChangeP1(v, addr+1, j+2); - sqlite3VdbeChangeP2(v, addr+1, addr+4); - sqlite3VdbeChangeP1(v, addr+3, j+2); - sqlite3VdbeChangeP2(v, addr+3, addr+2); - sqlite3VdbeJumpHere(v, addr+4); - sqlite3VdbeChangeP4(v, addr+6, - "wrong # of entries in index ", P4_STATIC); - sqlite3VdbeChangeP4(v, addr+7, pIdx->zName, P4_STATIC); - } - } - } - addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode); - sqlite3VdbeChangeP2(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr+1); - sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); - }else -#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ - -#ifndef SQLITE_OMIT_UTF16 - /* - ** PRAGMA encoding - ** PRAGMA encoding = "utf-8"|"utf-16"|"utf-16le"|"utf-16be" - ** - ** In its first form, this pragma returns the encoding of the main - ** database. If the database is not initialized, it is initialized now. - ** - ** The second form of this pragma is a no-op if the main database file - ** has not already been initialized. In this case it sets the default - ** encoding that will be used for the main database file if a new file - ** is created. If an existing main database file is opened, then the - ** default text encoding for the existing database is used. - ** - ** In all cases new databases created using the ATTACH command are - ** created to use the same default text encoding as the main database. If - ** the main database has not been initialized and/or created when ATTACH - ** is executed, this is done before the ATTACH operation. - ** - ** In the second form this pragma sets the text encoding to be used in - ** new database files created using this database handle. It is only - ** useful if invoked immediately after the main database i - */ - if( sqlite3StrICmp(zLeft, "encoding")==0 ){ - static const struct EncName { - char *zName; - u8 enc; - } encnames[] = { - { "UTF8", SQLITE_UTF8 }, - { "UTF-8", SQLITE_UTF8 }, /* Must be element [1] */ - { "UTF-16le", SQLITE_UTF16LE }, /* Must be element [2] */ - { "UTF-16be", SQLITE_UTF16BE }, /* Must be element [3] */ - { "UTF16le", SQLITE_UTF16LE }, - { "UTF16be", SQLITE_UTF16BE }, - { "UTF-16", 0 }, /* SQLITE_UTF16NATIVE */ - { "UTF16", 0 }, /* SQLITE_UTF16NATIVE */ - { 0, 0 } - }; - const struct EncName *pEnc; - if( !zRight ){ /* "PRAGMA encoding" */ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_String8, 0, 1); - assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); - assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); - assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - }else{ /* "PRAGMA encoding = XXX" */ - /* Only change the value of sqlite.enc if the database handle is not - ** initialized. If the main database exists, the new sqlite.enc value - ** will be overwritten when the schema is next loaded. If it does not - ** already exists, it will be created to use the new encoding value. - */ - if( - !(DbHasProperty(db, 0, DB_SchemaLoaded)) || - DbHasProperty(db, 0, DB_Empty) - ){ - for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ - if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; - break; - } - } - if( !pEnc->zName ){ - sqlite3ErrorMsg(pParse, "unsupported encoding: %s", zRight); - } - } - } - }else -#endif /* SQLITE_OMIT_UTF16 */ - -#ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS - /* - ** PRAGMA [database.]schema_version - ** PRAGMA [database.]schema_version = - ** - ** PRAGMA [database.]user_version - ** PRAGMA [database.]user_version = - ** - ** The pragma's schema_version and user_version are used to set or get - ** the value of the schema-version and user-version, respectively. Both - ** the schema-version and the user-version are 32-bit signed integers - ** stored in the database header. - ** - ** The schema-cookie is usually only manipulated internally by SQLite. It - ** is incremented by SQLite whenever the database schema is modified (by - ** creating or dropping a table or index). The schema version is used by - ** SQLite each time a query is executed to ensure that the internal cache - ** of the schema used when compiling the SQL query matches the schema of - ** the database against which the compiled query is actually executed. - ** Subverting this mechanism by using "PRAGMA schema_version" to modify - ** the schema-version is potentially dangerous and may lead to program - ** crashes or database corruption. Use with caution! - ** - ** The user-version is not used internally by SQLite. It may be used by - ** applications for any purpose. - */ - if( sqlite3StrICmp(zLeft, "schema_version")==0 - || sqlite3StrICmp(zLeft, "user_version")==0 - || sqlite3StrICmp(zLeft, "freelist_count")==0 - ){ - int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ - sqlite3VdbeUsesBtree(v, iDb); - switch( zLeft[0] ){ - case 'f': case 'F': - iCookie = BTREE_FREE_PAGE_COUNT; - break; - case 's': case 'S': - iCookie = BTREE_SCHEMA_VERSION; - break; - default: - iCookie = BTREE_USER_VERSION; - break; - } - - if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ - /* Write the specified cookie value */ - static const VdbeOpList setCookie[] = { - { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_Integer, 0, 1, 0}, /* 1 */ - { OP_SetCookie, 0, 0, 1}, /* 2 */ - }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); - sqlite3VdbeChangeP1(v, addr+2, iDb); - sqlite3VdbeChangeP2(v, addr+2, iCookie); - }else{ - /* Read the specified cookie value */ - static const VdbeOpList readCookie[] = { - { OP_Transaction, 0, 0, 0}, /* 0 */ - { OP_ReadCookie, 0, 1, 0}, /* 1 */ - { OP_ResultRow, 1, 1, 0} - }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie); - sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, iDb); - sqlite3VdbeChangeP3(v, addr+1, iCookie); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); - } - }else -#endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ - -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - /* - ** PRAGMA compile_options - ** - ** Return the names of all compile-time options used in this build, - ** one option per row. - */ - if( sqlite3StrICmp(zLeft, "compile_options")==0 ){ - int i = 0; - const char *zOpt; - sqlite3VdbeSetNumCols(v, 1); - pParse->nMem = 1; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); - while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } - }else -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ - -#ifndef SQLITE_OMIT_WAL - /* - ** PRAGMA [database.]wal_checkpoint - ** - ** Checkpoint the database. - */ - if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){ - if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeAddOp3(v, OP_Checkpoint, pId2->z?iDb:SQLITE_MAX_ATTACHED, 0, 0); - }else - - /* - ** PRAGMA wal_autocheckpoint - ** PRAGMA wal_autocheckpoint = N - ** - ** Configure a database connection to automatically checkpoint a database - ** after accumulating N frames in the log. Or query for the current value - ** of N. - */ - if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){ - if( zRight ){ - sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); - } - returnSingleInt(pParse, "wal_autocheckpoint", - db->xWalCallback==sqlite3WalDefaultHook ? - SQLITE_PTR_TO_INT(db->pWalArg) : 0); - }else -#endif - -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - /* - ** Report the current state of file logs for all databases - */ - if( sqlite3StrICmp(zLeft, "lock_status")==0 ){ - static const char *const azLockName[] = { - "unlocked", "shared", "reserved", "pending", "exclusive" - }; - int i; - sqlite3VdbeSetNumCols(v, 2); - pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); - for(i=0; inDb; i++){ - Btree *pBt; - Pager *pPager; - const char *zState = "unknown"; - int j; - if( db->aDb[i].zName==0 ) continue; - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); - pBt = db->aDb[i].pBt; - if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ - zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, - SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ - zState = azLockName[j]; - } - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); - } - - }else -#endif - -#ifdef SQLITE_HAS_CODEC - if( sqlite3StrICmp(zLeft, "key")==0 && zRight ){ - sqlite3_key(db, zRight, sqlite3Strlen30(zRight)); - }else - if( sqlite3StrICmp(zLeft, "rekey")==0 && zRight ){ - sqlite3_rekey(db, zRight, sqlite3Strlen30(zRight)); - }else - if( zRight && (sqlite3StrICmp(zLeft, "hexkey")==0 || - sqlite3StrICmp(zLeft, "hexrekey")==0) ){ - int i, h1, h2; - char zKey[40]; - for(i=0; (h1 = zRight[i])!=0 && (h2 = zRight[i+1])!=0; i+=2){ - h1 += 9*(1&(h1>>6)); - h2 += 9*(1&(h2>>6)); - zKey[i/2] = (h2 & 0x0f) | ((h1 & 0xf)<<4); - } - if( (zLeft[3] & 0xf)==0xb ){ - sqlite3_key(db, zKey, i/2); - }else{ - sqlite3_rekey(db, zKey, i/2); - } - }else -#endif -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ -#ifdef SQLITE_HAS_CODEC - if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ - sqlite3_activate_see(&zRight[4]); - } -#endif -#ifdef SQLITE_ENABLE_CEROD - if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){ - sqlite3_activate_cerod(&zRight[6]); - } -#endif - }else -#endif - - - {/* Empty ELSE clause */} - - /* - ** Reset the safety level, in case the fullfsync flag or synchronous - ** setting changed. - */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS - if( db->autoCommit ){ - sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, - (db->flags&SQLITE_FullFSync)!=0, - (db->flags&SQLITE_CkptFullFSync)!=0); + pNext = pFKey->pNextFrom; + sqlite3DbFree(db, pFKey); } -#endif -pragma_out: - sqlite3DbFree(db, zLeft); - sqlite3DbFree(db, zRight); } +#endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ -#endif /* SQLITE_OMIT_PRAGMA */ - -/************** End of pragma.c **********************************************/ -/************** Begin file prepare.c *****************************************/ +/************** End of fkey.c ************************************************/ +/************** Begin file insert.c ******************************************/ /* -** 2005 May 25 +** 2001 September 15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -86834,13505 +85026,18582 @@ pragma_out: ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the implementation of the sqlite3_prepare() -** interface, and routines that contribute to loading the database schema -** from disk. +** This file contains C code routines that are called by the parser +** to handle INSERT statements in SQLite. */ /* -** Fill the InitData structure with an error message that indicates -** that the database is corrupt. +** Generate code that will open a table for reading. */ -static void corruptSchema( - InitData *pData, /* Initialization context */ - const char *zObj, /* Object being parsed at the point of error */ - const char *zExtra /* Error information */ +SQLITE_PRIVATE void sqlite3OpenTable( + Parse *p, /* Generate code into this VDBE */ + int iCur, /* The cursor number of the table */ + int iDb, /* The database index in sqlite3.aDb[] */ + Table *pTab, /* The table to be opened */ + int opcode /* OP_OpenRead or OP_OpenWrite */ ){ - sqlite3 *db = pData->db; - if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ - if( zObj==0 ) zObj = "?"; - sqlite3SetString(pData->pzErrMsg, db, - "malformed database schema (%s)", zObj); - if( zExtra ){ - *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, - "%s - %s", *pData->pzErrMsg, zExtra); - } - } - pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT; + Vdbe *v; + if( IsVirtual(pTab) ) return; + v = sqlite3GetVdbe(p); + assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); + sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); + sqlite3VdbeAddOp3(v, opcode, iCur, pTab->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(pTab->nCol), P4_INT32); + VdbeComment((v, "%s", pTab->zName)); } /* -** This is the callback routine for the code that initializes the -** database. See sqlite3Init() below for additional information. -** This routine is also called from the OP_ParseSchema opcode of the VDBE. +** Return a pointer to the column affinity string associated with index +** pIdx. A column affinity string has one character for each column in +** the table, according to the affinity of the column: ** -** Each callback contains the following information: +** Character Column affinity +** ------------------------------ +** 'a' TEXT +** 'b' NONE +** 'c' NUMERIC +** 'd' INTEGER +** 'e' REAL ** -** argv[0] = name of thing being created -** argv[1] = root page number for table or index. 0 for trigger or view. -** argv[2] = SQL text for the CREATE statement. +** An extra 'b' is appended to the end of the string to cover the +** rowid that appears as the last column in every index. ** +** Memory for the buffer containing the column index affinity string +** is managed along with the rest of the Index structure. It will be +** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ - InitData *pData = (InitData*)pInit; - sqlite3 *db = pData->db; - int iDb = pData->iDb; - - assert( argc==3 ); - UNUSED_PARAMETER2(NotUsed, argc); - assert( sqlite3_mutex_held(db->mutex) ); - DbClearProperty(db, iDb, DB_Empty); - if( db->mallocFailed ){ - corruptSchema(pData, argv[0], 0); - return 1; - } - - assert( iDb>=0 && iDbnDb ); - if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ - if( argv[1]==0 ){ - corruptSchema(pData, argv[0], 0); - }else if( argv[2] && argv[2][0] ){ - /* Call the parser to process a CREATE TABLE, INDEX or VIEW. - ** But because db->init.busy is set to 1, no VDBE code is generated - ** or executed. All the parser does is build the internal data - ** structures that describe the table, index, or view. +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ + if( !pIdx->zColAff ){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. */ - int rc; - sqlite3_stmt *pStmt; - TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ - - assert( db->init.busy ); - db->init.iDb = iDb; - db->init.newTnum = sqlite3Atoi(argv[1]); - db->init.orphanTrigger = 0; - TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); - rc = db->errCode; - assert( (rc&0xFF)==(rcp&0xFF) ); - db->init.iDb = 0; - if( SQLITE_OK!=rc ){ - if( db->init.orphanTrigger ){ - assert( iDb==1 ); - }else{ - pData->rc = rc; - if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; - }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ - corruptSchema(pData, argv[0], sqlite3_errmsg(db)); - } - } + int n; + Table *pTab = pIdx->pTable; + sqlite3 *db = sqlite3VdbeDb(v); + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+2); + if( !pIdx->zColAff ){ + db->mallocFailed = 1; + return 0; } - sqlite3_finalize(pStmt); - }else if( argv[0]==0 ){ - corruptSchema(pData, 0, 0); - }else{ - /* If the SQL column is blank it means this is an index that - ** was created to be the PRIMARY KEY or to fulfill a UNIQUE - ** constraint for a CREATE TABLE. The index should have already - ** been created when we processed the CREATE TABLE. All we have - ** to do here is record the root page number for that index. - */ - Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); - if( pIndex==0 ){ - /* This can occur if there exists an index on a TEMP table which - ** has the same name as another index on a permanent index. Since - ** the permanent table is hidden by the TEMP table, we can also - ** safely ignore the index on the permanent table. - */ - /* Do Nothing */; - }else if( sqlite3GetInt32(argv[1], &pIndex->tnum)==0 ){ - corruptSchema(pData, argv[0], "invalid rootpage"); + for(n=0; nnColumn; n++){ + pIdx->zColAff[n] = pTab->aCol[pIdx->aiColumn[n]].affinity; } + pIdx->zColAff[n++] = SQLITE_AFF_NONE; + pIdx->zColAff[n] = 0; } - return 0; + + return pIdx->zColAff; } /* -** Attempt to read the database schema and initialize internal -** data structures for a single database file. The index of the -** database file is given by iDb. iDb==0 is used for the main -** database. iDb==1 should never be used. iDb>=2 is used for -** auxiliary databases. Return one of the SQLITE_ error codes to -** indicate success or failure. +** Set P4 of the most recently inserted opcode to a column affinity +** string for table pTab. A column affinity string has one character +** for each column indexed by the index, according to the affinity of the +** column: +** +** Character Column affinity +** ------------------------------ +** 'a' TEXT +** 'b' NONE +** 'c' NUMERIC +** 'd' INTEGER +** 'e' REAL */ -static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ - int rc; - int i; - int size; - Table *pTab; - Db *pDb; - char const *azArg[4]; - int meta[5]; - InitData initData; - char const *zMasterSchema; - char const *zMasterName = SCHEMA_TABLE(iDb); - int openedTransaction = 0; - - /* - ** The master database table has a structure like this - */ - static const char master_schema[] = - "CREATE TABLE sqlite_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#ifndef SQLITE_OMIT_TEMPDB - static const char temp_master_schema[] = - "CREATE TEMP TABLE sqlite_temp_master(\n" - " type text,\n" - " name text,\n" - " tbl_name text,\n" - " rootpage integer,\n" - " sql text\n" - ")" - ; -#else - #define temp_master_schema 0 -#endif - - assert( iDb>=0 && iDbnDb ); - assert( db->aDb[iDb].pSchema ); - assert( sqlite3_mutex_held(db->mutex) ); - assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - - /* zMasterSchema and zInitScript are set to point at the master schema - ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. - */ - if( !OMIT_TEMPDB && iDb==1 ){ - zMasterSchema = temp_master_schema; - }else{ - zMasterSchema = master_schema; - } - zMasterName = SCHEMA_TABLE(iDb); - - /* Construct the schema tables. */ - azArg[0] = zMasterName; - azArg[1] = "1"; - azArg[2] = zMasterSchema; - azArg[3] = 0; - initData.db = db; - initData.iDb = iDb; - initData.rc = SQLITE_OK; - initData.pzErrMsg = pzErrMsg; - sqlite3InitCallback(&initData, 3, (char **)azArg, 0); - if( initData.rc ){ - rc = initData.rc; - goto error_out; - } - pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); - if( ALWAYS(pTab) ){ - pTab->tabFlags |= TF_Readonly; - } - - /* Create a cursor to hold the database open - */ - pDb = &db->aDb[iDb]; - if( pDb->pBt==0 ){ - if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){ - DbSetProperty(db, 1, DB_SchemaLoaded); - } - return SQLITE_OK; - } - - /* If there is not already a read-only (or read-write) transaction opened - ** on the b-tree database, open one now. If a transaction is opened, it - ** will be closed before this function returns. */ - sqlite3BtreeEnter(pDb->pBt); - if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ - rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); - if( rc!=SQLITE_OK ){ - sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); - goto initone_error_out; - } - openedTransaction = 1; - } - - /* Get the database meta information. - ** - ** Meta values are as follows: - ** meta[0] Schema cookie. Changes with each schema change. - ** meta[1] File format of schema layer. - ** meta[2] Size of the page cache. - ** meta[3] Largest rootpage (auto/incr_vacuum mode) - ** meta[4] Db text encoding. 1:UTF-8 2:UTF-16LE 3:UTF-16BE - ** meta[5] User version - ** meta[6] Incremental vacuum mode - ** meta[7] unused - ** meta[8] unused - ** meta[9] unused +SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ + /* The first time a column affinity string for a particular table + ** is required, it is allocated and populated here. It is then + ** stored as a member of the Table structure for subsequent use. ** - ** Note: The #defined SQLITE_UTF* symbols in sqliteInt.h correspond to - ** the possible values of meta[4]. + ** The column affinity string will eventually be deleted by + ** sqlite3DeleteTable() when the Table structure itself is cleaned up. */ - for(i=0; ipBt, i+1, (u32 *)&meta[i]); - } - pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1]; + if( !pTab->zColAff ){ + char *zColAff; + int i; + sqlite3 *db = sqlite3VdbeDb(v); - /* If opening a non-empty database, check the text encoding. For the - ** main database, set sqlite3.enc to the encoding of the main database. - ** For an attached db, it is an error if the encoding is not the same - ** as sqlite3.enc. - */ - if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ - if( iDb==0 ){ - u8 encoding; - /* If opening the main database, set ENC(db). */ - encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; - if( encoding==0 ) encoding = SQLITE_UTF8; - ENC(db) = encoding; - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); - }else{ - /* If opening an attached database, the encoding much match ENC(db) */ - if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ - sqlite3SetString(pzErrMsg, db, "attached databases must use the same" - " text encoding as main database"); - rc = SQLITE_ERROR; - goto initone_error_out; - } + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + if( !zColAff ){ + db->mallocFailed = 1; + return; } - }else{ - DbSetProperty(db, iDb, DB_Empty); - } - pDb->pSchema->enc = ENC(db); - if( pDb->pSchema->cache_size==0 ){ - size = meta[BTREE_DEFAULT_CACHE_SIZE-1]; - if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } - if( size<0 ) size = -size; - pDb->pSchema->cache_size = size; - sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); - } + for(i=0; inCol; i++){ + zColAff[i] = pTab->aCol[i].affinity; + } + zColAff[pTab->nCol] = '\0'; - /* - ** file_format==1 Version 3.0.0. - ** file_format==2 Version 3.1.3. // ALTER TABLE ADD COLUMN - ** file_format==3 Version 3.1.4. // ditto but with non-NULL defaults - ** file_format==4 Version 3.3.0. // DESC indices. Boolean constants - */ - pDb->pSchema->file_format = (u8)meta[BTREE_FILE_FORMAT-1]; - if( pDb->pSchema->file_format==0 ){ - pDb->pSchema->file_format = 1; - } - if( pDb->pSchema->file_format>SQLITE_MAX_FILE_FORMAT ){ - sqlite3SetString(pzErrMsg, db, "unsupported file format"); - rc = SQLITE_ERROR; - goto initone_error_out; + pTab->zColAff = zColAff; } - /* Ticket #2804: When we open a database in the newer file format, - ** clear the legacy_file_format pragma flag so that a VACUUM will - ** not downgrade the database and thus invalidate any descending - ** indices that the user might have created. - */ - if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ - db->flags &= ~SQLITE_LegacyFileFmt; - } + sqlite3VdbeChangeP4(v, -1, pTab->zColAff, P4_TRANSIENT); +} - /* Read the schema information out of the schema tables - */ - assert( db->init.busy ); - { - char *zSql; - zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", - db->aDb[iDb].zName, zMasterName); -#ifndef SQLITE_OMIT_AUTHORIZATION - { - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); - xAuth = db->xAuth; - db->xAuth = 0; +/* +** Return non-zero if the table pTab in database iDb or any of its indices +** have been opened at any point in the VDBE program beginning at location +** iStartAddr throught the end of the program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using temporary table for the results of the SELECT. +*/ +static int readsTable(Parse *p, int iStartAddr, int iDb, Table *pTab){ + Vdbe *v = sqlite3GetVdbe(p); + int i; + int iEnd = sqlite3VdbeCurrentAddr(v); +#ifndef SQLITE_OMIT_VIRTUALTABLE + VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; #endif - rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); -#ifndef SQLITE_OMIT_AUTHORIZATION - db->xAuth = xAuth; + + for(i=iStartAddr; iopcode==OP_OpenRead && pOp->p3==iDb ){ + Index *pIndex; + int tnum = pOp->p2; + if( tnum==pTab->tnum ){ + return 1; + } + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( tnum==pIndex->tnum ){ + return 1; + } + } } -#endif - if( rc==SQLITE_OK ) rc = initData.rc; - sqlite3DbFree(db, zSql); -#ifndef SQLITE_OMIT_ANALYZE - if( rc==SQLITE_OK ){ - sqlite3AnalysisLoad(db, iDb); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pOp->opcode==OP_VOpen && pOp->p4.pVtab==pVTab ){ + assert( pOp->p4.pVtab!=0 ); + assert( pOp->p4type==P4_VTAB ); + return 1; } #endif } - if( db->mallocFailed ){ - rc = SQLITE_NOMEM; - sqlite3ResetInternalSchema(db, 0); - } - if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ - /* Black magic: If the SQLITE_RecoveryMode flag is set, then consider - ** the schema loaded, even if errors occurred. In this situation the - ** current sqlite3_prepare() operation will fail, but the following one - ** will attempt to compile the supplied statement against whatever subset - ** of the schema was loaded before the error occurred. The primary - ** purpose of this is to allow access to the sqlite_master table - ** even when its contents have been corrupted. - */ - DbSetProperty(db, iDb, DB_SchemaLoaded); - rc = SQLITE_OK; - } - - /* Jump here for an error that occurs after successfully allocating - ** curMain and calling sqlite3BtreeEnter(). For an error that occurs - ** before that point, jump to error_out. - */ -initone_error_out: - if( openedTransaction ){ - sqlite3BtreeCommit(pDb->pBt); - } - sqlite3BtreeLeave(pDb->pBt); - -error_out: - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; - } - return rc; + return 0; } +#ifndef SQLITE_OMIT_AUTOINCREMENT /* -** Initialize all database files - the main database file, the file -** used to store temporary tables, and any additional database files -** created using ATTACH statements. Return a success code. If an -** error occurs, write an error message into *pzErrMsg. +** Locate or create an AutoincInfo structure associated with table pTab +** which is in database iDb. Return the register number for the register +** that holds the maximum rowid. ** -** After a database is initialized, the DB_SchemaLoaded bit is set -** bit is set in the flags field of the Db structure. If the database -** file was of zero-length, then the DB_Empty flag is also set. +** There is at most one AutoincInfo structure per table even if the +** same table is autoincremented multiple times due to inserts within +** triggers. A new AutoincInfo structure is created if this is the +** first use of table pTab. On 2nd and subsequent uses, the original +** AutoincInfo structure is used. +** +** Three memory locations are allocated: +** +** (1) Register to hold the name of the pTab table. +** (2) Register to hold the maximum ROWID of pTab. +** (3) Register to hold the rowid in sqlite_sequence of pTab +** +** The 2nd register is the one that is returned. That is all the +** insert routine needs to know about. */ -SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ - int i, rc; - int commit_internal = !(db->flags&SQLITE_InternChanges); - - assert( sqlite3_mutex_held(db->mutex) ); - rc = SQLITE_OK; - db->init.busy = 1; - for(i=0; rc==SQLITE_OK && inDb; i++){ - if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; - rc = sqlite3InitOne(db, i, pzErrMsg); - if( rc ){ - sqlite3ResetInternalSchema(db, i); - } - } +static int autoIncBegin( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database holding pTab */ + Table *pTab /* The table we are writing to */ +){ + int memId = 0; /* Register holding maximum rowid */ + if( pTab->tabFlags & TF_Autoincrement ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + AutoincInfo *pInfo; - /* Once all the other databases have been initialised, load the schema - ** for the TEMP database. This is loaded last, as the TEMP database - ** schema may contain references to objects in other databases. - */ -#ifndef SQLITE_OMIT_TEMPDB - if( rc==SQLITE_OK && ALWAYS(db->nDb>1) - && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, 1, pzErrMsg); - if( rc ){ - sqlite3ResetInternalSchema(db, 1); + pInfo = pToplevel->pAinc; + while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } + if( pInfo==0 ){ + pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); + if( pInfo==0 ) return 0; + pInfo->pNext = pToplevel->pAinc; + pToplevel->pAinc = pInfo; + pInfo->pTab = pTab; + pInfo->iDb = iDb; + pToplevel->nMem++; /* Register to hold name of table */ + pInfo->regCtr = ++pToplevel->nMem; /* Max rowid register */ + pToplevel->nMem++; /* Rowid in sqlite_sequence */ } + memId = pInfo->regCtr; } -#endif + return memId; +} - db->init.busy = 0; - if( rc==SQLITE_OK && commit_internal ){ - sqlite3CommitInternalChanges(db); - } +/* +** This routine generates code that will initialize all of the +** register used by the autoincrement tracker. +*/ +SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ + AutoincInfo *p; /* Information about an AUTOINCREMENT */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* Database only autoinc table */ + int memId; /* Register holding max rowid */ + int addr; /* A VDBE address */ + Vdbe *v = pParse->pVdbe; /* VDBE under construction */ - return rc; + /* This routine is never called during trigger-generation. It is + ** only called from the top-level */ + assert( pParse->pTriggerTab==0 ); + assert( pParse==sqlite3ParseToplevel(pParse) ); + + assert( v ); /* We failed long ago if this is not so */ + for(p = pParse->pAinc; p; p = p->pNext){ + pDb = &db->aDb[p->iDb]; + memId = p->regCtr; + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); + sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); + sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); + sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); + sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); + sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); + sqlite3VdbeAddOp0(v, OP_Close); + } } /* -** This routine is a no-op if the database schema is already initialised. -** Otherwise, the schema is loaded. An error code is returned. +** Update the maximum rowid for an autoincrement calculation. +** +** This routine should be called when the top of the stack holds a +** new rowid that is about to be inserted. If that new rowid is +** larger than the maximum rowid in the memId memory cell, then the +** memory cell is updated. The stack is unchanged. */ -SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ - int rc = SQLITE_OK; - sqlite3 *db = pParse->db; - assert( sqlite3_mutex_held(db->mutex) ); - if( !db->init.busy ){ - rc = sqlite3Init(db, &pParse->zErrMsg); - } - if( rc!=SQLITE_OK ){ - pParse->rc = rc; - pParse->nErr++; +static void autoIncStep(Parse *pParse, int memId, int regRowid){ + if( memId>0 ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); } - return rc; } - /* -** Check schema cookies in all databases. If any cookie is out -** of date set pParse->rc to SQLITE_SCHEMA. If all schema cookies -** make no changes to pParse->rc. +** This routine generates the code needed to write autoincrement +** maximum rowid values back into the sqlite_sequence register. +** Every statement that might do an INSERT into an autoincrement +** table (either directly or through triggers) needs to call this +** routine just before the "exit" code. */ -static void schemaIsValid(Parse *pParse){ +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ + AutoincInfo *p; + Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; - int iDb; - int rc; - int cookie; - - assert( pParse->checkSchema ); - assert( sqlite3_mutex_held(db->mutex) ); - for(iDb=0; iDbnDb; iDb++){ - int openedTransaction = 0; /* True if a transaction is opened */ - Btree *pBt = db->aDb[iDb].pBt; /* Btree database to read cookie from */ - if( pBt==0 ) continue; - - /* If there is not already a read-only (or read-write) transaction opened - ** on the b-tree database, open one now. If a transaction is opened, it - ** will be closed immediately after reading the meta-value. */ - if( !sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; - } - if( rc!=SQLITE_OK ) return; - openedTransaction = 1; - } - /* Read the schema cookie from the database. If it does not match the - ** value stored as part of the in-memory schema representation, - ** set Parse.rc to SQLITE_SCHEMA. */ - sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); - if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ - pParse->rc = SQLITE_SCHEMA; - } + assert( v ); + for(p = pParse->pAinc; p; p = p->pNext){ + Db *pDb = &db->aDb[p->iDb]; + int j1, j2, j3, j4, j5; + int iRec; + int memId = p->regCtr; - /* Close the transaction, if one was opened. */ - if( openedTransaction ){ - sqlite3BtreeCommit(pBt); - } + iRec = sqlite3GetTempReg(pParse); + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); + j2 = sqlite3VdbeAddOp0(v, OP_Rewind); + j3 = sqlite3VdbeAddOp3(v, OP_Column, 0, 0, iRec); + j4 = sqlite3VdbeAddOp3(v, OP_Eq, memId-1, 0, iRec); + sqlite3VdbeAddOp2(v, OP_Next, 0, j3); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); + j5 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, j4); + sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, j5); + sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); + sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); + sqlite3ReleaseTempReg(pParse, iRec); } } - +#else /* -** Convert a schema pointer into the iDb index that indicates -** which database file in db->aDb[] the schema refers to. -** -** If the same database is attached more than once, the first -** attached database is returned. +** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines +** above are all no-ops */ -SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ - int i = -1000000; +# define autoIncBegin(A,B,C) (0) +# define autoIncStep(A,B,C) +#endif /* SQLITE_OMIT_AUTOINCREMENT */ - /* If pSchema is NULL, then return -1000000. This happens when code in - ** expr.c is trying to resolve a reference to a transient table (i.e. one - ** created by a sub-select). In this case the return value of this - ** function should never be used. - ** - ** We return -1000000 instead of the more usual -1 simply because using - ** -1000000 as the incorrect index into db->aDb[] is much - ** more likely to cause a segfault than -1 (of course there are assert() - ** statements too, but it never hurts to play the odds). - */ - assert( sqlite3_mutex_held(db->mutex) ); - if( pSchema ){ - for(i=0; ALWAYS(inDb); i++){ - if( db->aDb[i].pSchema==pSchema ){ - break; - } - } - assert( i>=0 && inDb ); - } - return i; -} + +/* Forward declaration */ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +); /* -** Compile the UTF-8 encoded SQL statement zSql into a statement handle. +** This routine is call to handle SQL of the following forms: +** +** insert into TABLE (IDLIST) values(EXPRLIST) +** insert into TABLE (IDLIST) select +** +** The IDLIST following the table name is always optional. If omitted, +** then a list of all columns for the table is substituted. The IDLIST +** appears in the pColumn parameter. pColumn is NULL if IDLIST is omitted. +** +** The pList parameter holds EXPRLIST in the first form of the INSERT +** statement above, and pSelect is NULL. For the second form, pList is +** NULL and pSelect is a pointer to the select statement used to generate +** data for the insert. +** +** The code generated follows one of four templates. For a simple +** select with data coming from a VALUES clause, the code executes +** once straight down through. Pseudo-code follows (we call this +** the "1st template"): +** +** open write cursor to
    and its indices +** puts VALUES clause expressions onto the stack +** write the resulting record into
    +** cleanup +** +** The three remaining templates assume the statement is of the form +** +** INSERT INTO
    SELECT ... +** +** If the SELECT clause is of the restricted form "SELECT * FROM " - +** in other words if the SELECT pulls all columns from a single table +** and there is no WHERE or LIMIT or GROUP BY or ORDER BY clauses, and +** if and are distinct tables but have identical +** schemas, including all the same indices, then a special optimization +** is invoked that copies raw records from over to . +** See the xferOptimization() function for the implementation of this +** template. This is the 2nd template. +** +** open a write cursor to
    +** open read cursor on +** transfer all records in over to
    +** close cursors +** foreach index on
    +** open a write cursor on the
    index +** open a read cursor on the corresponding index +** transfer all records from the read to the write cursors +** close cursors +** end foreach +** +** The 3rd template is for when the second template does not apply +** and the SELECT clause does not read from
    at any time. +** The generated code follows this template: +** +** EOF <- 0 +** X <- A +** goto B +** A: setup for the SELECT +** loop over the rows in the SELECT +** load values into registers R..R+n +** yield X +** end loop +** cleanup after the SELECT +** EOF <- 1 +** yield X +** goto A +** B: open write cursor to
    and its indices +** C: yield X +** if EOF goto D +** insert the select result into
    from R..R+n +** goto C +** D: cleanup +** +** The 4th template is used if the insert statement takes its +** values from a SELECT but the data is being inserted into a table +** that is also read as part of the SELECT. In the third form, +** we have to use a intermediate table to store the results of +** the select. The template is like this: +** +** EOF <- 0 +** X <- A +** goto B +** A: setup for the SELECT +** loop over the tables in the SELECT +** load value into register R..R+n +** yield X +** end loop +** cleanup after the SELECT +** EOF <- 1 +** yield X +** halt-error +** B: open temp table +** L: yield X +** if EOF goto M +** insert row from R..R+n into temp table +** goto L +** M: open write cursor to
    and its indices +** rewind temp table +** C: loop over rows of intermediate table +** transfer values form intermediate table into
    +** end loop +** D: cleanup */ -static int sqlite3Prepare( - sqlite3 *db, /* Database handle. */ - const char *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ - Vdbe *pReprepare, /* VM being reprepared */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const char **pzTail /* OUT: End of parsed string */ +SQLITE_PRIVATE void sqlite3Insert( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* Name of table into which we are inserting */ + ExprList *pList, /* List of values to be inserted */ + Select *pSelect, /* A SELECT statement to use as the data source */ + IdList *pColumn, /* Column names corresponding to IDLIST. */ + int onError /* How to handle constraint errors */ ){ - Parse *pParse; /* Parsing context */ - char *zErrMsg = 0; /* Error message */ - int rc = SQLITE_OK; /* Result code */ - int i; /* Loop counter */ - - /* Allocate the parsing context */ - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; - goto end_prepare; - } - pParse->pReprepare = pReprepare; - assert( ppStmt && *ppStmt==0 ); - assert( !db->mallocFailed ); - assert( sqlite3_mutex_held(db->mutex) ); + sqlite3 *db; /* The main database structure */ + Table *pTab; /* The table to insert into. aka TABLE */ + char *zTab; /* Name of the table into which we are inserting */ + const char *zDb; /* Name of the database holding this table */ + int i, j, idx; /* Loop counters */ + Vdbe *v; /* Generate code into this virtual machine */ + Index *pIdx; /* For looping over indices of the table */ + int nColumn; /* Number of columns in the data */ + int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ + int baseCur = 0; /* VDBE Cursor number for pTab */ + int keyColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ + int endOfLoop; /* Label for the end of the insertion loop */ + int useTempTable = 0; /* Store SELECT results in intermediate table */ + int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ + int addrInsTop = 0; /* Jump to label "D" */ + int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ + int addrSelect = 0; /* Address of coroutine that implements the SELECT */ + SelectDest dest; /* Destination for SELECT on rhs of INSERT */ + int iDb; /* Index of database holding TABLE */ + Db *pDb; /* The database containing table being inserted into */ + int appendFlag = 0; /* True if the insert is likely to be an append */ - /* Check to verify that it is possible to get a read lock on all - ** database schemas. The inability to get a read lock indicates that - ** some other database connection is holding a write-lock, which in - ** turn means that the other connection has made uncommitted changes - ** to the schema. - ** - ** Were we to proceed and prepare the statement against the uncommitted - ** schema changes and if those schema changes are subsequently rolled - ** back and different changes are made in their place, then when this - ** prepared statement goes to run the schema cookie would fail to detect - ** the schema change. Disaster would follow. - ** - ** This thread is currently holding mutexes on all Btrees (because - ** of the sqlite3BtreeEnterAll() in sqlite3LockAndPrepare()) so it - ** is not possible for another thread to start a new schema change - ** while this routine is running. Hence, we do not need to hold - ** locks on the schema, we just need to make sure nobody else is - ** holding them. - ** - ** Note that setting READ_UNCOMMITTED overrides most lock detection, - ** but it does *not* override schema lock detection, so this all still - ** works even if READ_UNCOMMITTED is set. - */ - for(i=0; inDb; i++) { - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - assert( sqlite3BtreeHoldsMutex(pBt) ); - rc = sqlite3BtreeSchemaLocked(pBt); - if( rc ){ - const char *zDb = db->aDb[i].zName; - sqlite3Error(db, rc, "database schema is locked: %s", zDb); - testcase( db->flags & SQLITE_ReadUncommitted ); - goto end_prepare; - } - } - } + /* Register allocations */ + int regFromSelect = 0;/* Base register for data coming from SELECT */ + int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ + int regRowCount = 0; /* Memory cell used for the row counter */ + int regIns; /* Block of regs holding rowid+data being inserted */ + int regRowid; /* registers holding insert rowid */ + int regData; /* register holding first column to insert */ + int regEof = 0; /* Register recording end of SELECT data */ + int *aRegIdx = 0; /* One register allocated to each index */ - sqlite3VtabUnlockList(db); +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to insert into a view */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of trigger times */ +#endif - pParse->db = db; - pParse->nQueryLoop = (double)1; - if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ - char *zSqlCopy; - int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - testcase( nBytes==mxLen ); - testcase( nBytes==mxLen+1 ); - if( nBytes>mxLen ){ - sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); - rc = sqlite3ApiExit(db, SQLITE_TOOBIG); - goto end_prepare; - } - zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); - if( zSqlCopy ){ - sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); - sqlite3DbFree(db, zSqlCopy); - pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; - }else{ - pParse->zTail = &zSql[nBytes]; - } - }else{ - sqlite3RunParser(pParse, zSql, &zErrMsg); + db = pParse->db; + memset(&dest, 0, sizeof(dest)); + if( pParse->nErr || db->mallocFailed ){ + goto insert_cleanup; } - assert( 1==(int)pParse->nQueryLoop ); - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; - } - if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; - if( pParse->checkSchema ){ - schemaIsValid(pParse); - } - if( pParse->rc==SQLITE_SCHEMA ){ - sqlite3ResetInternalSchema(db, 0); - } - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + /* Locate the table into which we will be inserting new information. + */ + assert( pTabList->nSrc==1 ); + zTab = pTabList->a[0].zName; + if( NEVER(zTab==0) ) goto insert_cleanup; + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ){ + goto insert_cleanup; } - if( pzTail ){ - *pzTail = pParse->zTail; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + pDb = &db->aDb[iDb]; + zDb = pDb->zName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ + goto insert_cleanup; } - rc = pParse->rc; -#ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ - static const char * const azColName[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "selectid", "order", "from", "detail" - }; - int iFirst, mx; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 4); - iFirst = 8; - mx = 12; - }else{ - sqlite3VdbeSetNumCols(pParse->pVdbe, 8); - iFirst = 0; - mx = 8; - } - for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, - azColName[i], SQLITE_STATIC); - } - } + /* Figure out if we have any triggers and if the table being + ** inserted into is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); + isView = pTab->pSelect!=0; +#else +# define pTrigger 0 +# define tmask 0 +# define isView 0 #endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); - assert( db->init.busy==0 || saveSqlFlag==0 ); - if( db->init.busy==0 ){ - Vdbe *pVdbe = pParse->pVdbe; - sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); - } - if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(pParse->pVdbe); - assert(!(*ppStmt)); - }else{ - *ppStmt = (sqlite3_stmt*)pParse->pVdbe; - } - - if( zErrMsg ){ - sqlite3Error(db, rc, "%s", zErrMsg); - sqlite3DbFree(db, zErrMsg); - }else{ - sqlite3Error(db, rc, 0); + /* If pTab is really a view, make sure it has been initialized. + ** ViewGetColumnNames() is a no-op if pTab is not a view (or virtual + ** module table). + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto insert_cleanup; } - /* Delete any TriggerPrg structures allocated while parsing this statement. */ - while( pParse->pTriggerPrg ){ - TriggerPrg *pT = pParse->pTriggerPrg; - pParse->pTriggerPrg = pT->pNext; - sqlite3DbFree(db, pT); + /* Ensure that: + * (a) the table is not read-only, + * (b) that if it is a view then ON INSERT triggers exist + */ + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto insert_cleanup; } -end_prepare: + /* Allocate a VDBE + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto insert_cleanup; + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, pSelect || pTrigger, iDb); - sqlite3StackFree(db, pParse); - rc = sqlite3ApiExit(db, rc); - assert( (rc&db->errMask)==rc ); - return rc; -} -static int sqlite3LockAndPrepare( - sqlite3 *db, /* Database handle. */ - const char *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ - Vdbe *pOld, /* VM being reprepared */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const char **pzTail /* OUT: End of parsed string */ -){ - int rc; - assert( ppStmt!=0 ); - *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); - if( rc==SQLITE_SCHEMA ){ - sqlite3_finalize(*ppStmt); - rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); +#ifndef SQLITE_OMIT_XFER_OPT + /* If the statement is of the form + ** + ** INSERT INTO SELECT * FROM ; + ** + ** Then special optimizations can be applied that make the transfer + ** very fast and which reduce fragmentation of indices. + ** + ** This is the 2nd template. + */ + if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + assert( !pTrigger ); + assert( pList==0 ); + goto insert_end; } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return rc; -} +#endif /* SQLITE_OMIT_XFER_OPT */ -/* -** Rerun the compilation of a statement after a schema change. -** -** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, -** if the statement cannot be recompiled because another connection has -** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error -** occurs, return SQLITE_SCHEMA. -*/ -SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ - int rc; - sqlite3_stmt *pNew; - const char *zSql; - sqlite3 *db; + /* If this is an AUTOINCREMENT table, look up the sequence number in the + ** sqlite_sequence table and store it in memory cell regAutoinc. + */ + regAutoinc = autoIncBegin(pParse, iDb, pTab); - assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); - zSql = sqlite3_sql((sqlite3_stmt *)p); - assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ - db = sqlite3VdbeDb(p); - assert( sqlite3_mutex_held(db->mutex) ); - rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); - if( rc ){ - if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; - } - assert( pNew==0 ); - return rc; - }else{ - assert( pNew!=0 ); - } - sqlite3VdbeSwap((Vdbe*)pNew, p); - sqlite3TransferBindings(pNew, (sqlite3_stmt*)p); - sqlite3VdbeResetStepResult((Vdbe*)pNew); - sqlite3VdbeFinalize((Vdbe*)pNew); - return SQLITE_OK; -} - - -/* -** Two versions of the official API. Legacy and new use. In the legacy -** version, the original SQL text is not saved in the prepared statement -** and so if a schema change occurs, SQLITE_SCHEMA is returned by -** sqlite3_step(). In the new version, the original SQL text is retained -** and the statement is automatically recompiled if an schema change -** occurs. -*/ -SQLITE_API int sqlite3_prepare( - sqlite3 *db, /* Database handle. */ - const char *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const char **pzTail /* OUT: End of parsed string */ -){ - int rc; - rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,0,ppStmt,pzTail); - assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ - return rc; -} -SQLITE_API int sqlite3_prepare_v2( - sqlite3 *db, /* Database handle. */ - const char *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const char **pzTail /* OUT: End of parsed string */ -){ - int rc; - rc = sqlite3LockAndPrepare(db,zSql,nBytes,1,0,ppStmt,pzTail); - assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ - return rc; -} - - -#ifndef SQLITE_OMIT_UTF16 -/* -** Compile the UTF-16 encoded SQL statement zSql into a statement handle. -*/ -static int sqlite3Prepare16( - sqlite3 *db, /* Database handle. */ - const void *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - int saveSqlFlag, /* True to save SQL text into the sqlite3_stmt */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const void **pzTail /* OUT: End of parsed string */ -){ - /* This function currently works by first transforming the UTF-16 - ** encoded string to UTF-8, then invoking sqlite3_prepare(). The - ** tricky bit is figuring out the pointer to return in *pzTail. + /* Figure out how many columns of data are supplied. If the data + ** is coming from a SELECT statement, then generate a co-routine that + ** produces a single row of the SELECT on each invocation. The + ** co-routine is the common header to the 3rd and 4th templates. */ - char *zSql8; - const char *zTail8 = 0; - int rc = SQLITE_OK; - - assert( ppStmt ); - *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - sqlite3_mutex_enter(db->mutex); - zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); - if( zSql8 ){ - rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, 0, ppStmt, &zTail8); - } - - if( zTail8 && pzTail ){ - /* If sqlite3_prepare returns a tail pointer, we calculate the - ** equivalent pointer into the UTF-16 string by counting the unicode - ** characters between zSql8 and zTail8, and then returning a pointer - ** the same number of characters into the UTF-16 string. + if( pSelect ){ + /* Data is coming from a SELECT. Generate code to implement that SELECT + ** as a co-routine. The code is common to both the 3rd and 4th + ** templates: + ** + ** EOF <- 0 + ** X <- A + ** goto B + ** A: setup for the SELECT + ** loop over the tables in the SELECT + ** load value into register R..R+n + ** yield X + ** end loop + ** cleanup after the SELECT + ** EOF <- 1 + ** yield X + ** halt-error + ** + ** On each invocation of the co-routine, it puts a single row of the + ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. + ** (These output registers are allocated by sqlite3Select().) When + ** the SELECT completes, it sets the EOF flag stored in regEof. */ - int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); - *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); - } - sqlite3DbFree(db, zSql8); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -/* -** Two versions of the official API. Legacy and new use. In the legacy -** version, the original SQL text is not saved in the prepared statement -** and so if a schema change occurs, SQLITE_SCHEMA is returned by -** sqlite3_step(). In the new version, the original SQL text is retained -** and the statement is automatically recompiled if an schema change -** occurs. -*/ -SQLITE_API int sqlite3_prepare16( - sqlite3 *db, /* Database handle. */ - const void *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const void **pzTail /* OUT: End of parsed string */ -){ - int rc; - rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); - assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ - return rc; -} -SQLITE_API int sqlite3_prepare16_v2( - sqlite3 *db, /* Database handle. */ - const void *zSql, /* UTF-8 encoded SQL statement. */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ - const void **pzTail /* OUT: End of parsed string */ -){ - int rc; - rc = sqlite3Prepare16(db,zSql,nBytes,1,ppStmt,pzTail); - assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ - return rc; -} - -#endif /* SQLITE_OMIT_UTF16 */ + int rc, j1; -/************** End of prepare.c *********************************************/ -/************** Begin file select.c ******************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains C code routines that are called by the parser -** to handle SELECT statements in SQLite. -*/ + regEof = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ + VdbeComment((v, "SELECT eof flag")); + sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); + addrSelect = sqlite3VdbeCurrentAddr(v)+2; + sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iParm); + j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + VdbeComment((v, "Jump over SELECT coroutine")); + /* Resolve the expressions in the SELECT statement and execute it. */ + rc = sqlite3Select(pParse, pSelect, &dest); + assert( pParse->nErr==0 || rc ); + if( rc || NEVER(pParse->nErr) || db->mallocFailed ){ + goto insert_cleanup; + } + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ + sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); /* yield X */ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); + VdbeComment((v, "End of SELECT coroutine")); + sqlite3VdbeJumpHere(v, j1); /* label B: */ -/* -** Delete all the content of a Select structure but do not deallocate -** the select structure itself. -*/ -static void clearSelect(sqlite3 *db, Select *p){ - sqlite3ExprListDelete(db, p->pEList); - sqlite3SrcListDelete(db, p->pSrc); - sqlite3ExprDelete(db, p->pWhere); - sqlite3ExprListDelete(db, p->pGroupBy); - sqlite3ExprDelete(db, p->pHaving); - sqlite3ExprListDelete(db, p->pOrderBy); - sqlite3SelectDelete(db, p->pPrior); - sqlite3ExprDelete(db, p->pLimit); - sqlite3ExprDelete(db, p->pOffset); -} + regFromSelect = dest.iMem; + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + assert( dest.nMem==nColumn ); -/* -** Initialize a SelectDest structure. -*/ -SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ - pDest->eDest = (u8)eDest; - pDest->iParm = iParm; - pDest->affinity = 0; - pDest->iMem = 0; - pDest->nMem = 0; -} + /* Set useTempTable to TRUE if the result of the SELECT statement + ** should be written into a temporary table (template 4). Set to + ** FALSE if each* row of the SELECT can be written directly into + ** the destination table (template 3). + ** + ** A temp table must be used if the table being updated is also one + ** of the tables being read by the SELECT statement. Also use a + ** temp table in the case of row triggers. + */ + if( pTrigger || readsTable(pParse, addrSelect, iDb, pTab) ){ + useTempTable = 1; + } + if( useTempTable ){ + /* Invoke the coroutine to extract information from the SELECT + ** and add it to a transient table srcTab. The code generated + ** here is from the 4th template: + ** + ** B: open temp table + ** L: yield X + ** if EOF goto M + ** insert row from R..R+n into temp table + ** goto L + ** M: ... + */ + int regRec; /* Register to hold packed record */ + int regTempRowid; /* Register to hold temp table ROWID */ + int addrTop; /* Label "L" */ + int addrIf; /* Address of jump to M */ -/* -** Allocate a new Select structure and return a pointer to that -** structure. -*/ -SQLITE_PRIVATE Select *sqlite3SelectNew( - Parse *pParse, /* Parsing context */ - ExprList *pEList, /* which columns to include in the result */ - SrcList *pSrc, /* the FROM clause -- which tables to scan */ - Expr *pWhere, /* the WHERE clause */ - ExprList *pGroupBy, /* the GROUP BY clause */ - Expr *pHaving, /* the HAVING clause */ - ExprList *pOrderBy, /* the ORDER BY clause */ - int isDistinct, /* true if the DISTINCT keyword is present */ - Expr *pLimit, /* LIMIT value. NULL means not used */ - Expr *pOffset /* OFFSET value. NULL means no offset */ -){ - Select *pNew; - Select standin; - sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); - assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ - if( pNew==0 ){ - pNew = &standin; - memset(pNew, 0, sizeof(*pNew)); + srcTab = pParse->nTab++; + regRec = sqlite3GetTempReg(pParse); + regTempRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); + addrIf = sqlite3VdbeAddOp1(v, OP_If, regEof); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); + sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeJumpHere(v, addrIf); + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempReg(pParse, regTempRowid); + } + }else{ + /* This is the case if the data for the INSERT is coming from a VALUES + ** clause + */ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + srcTab = -1; + assert( useTempTable==0 ); + nColumn = pList ? pList->nExpr : 0; + for(i=0; ia[i].pExpr) ){ + goto insert_cleanup; + } + } } - if( pEList==0 ){ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); + + /* Make sure the number of columns in the source data matches the number + ** of columns to be inserted into the table. + */ + if( IsVirtual(pTab) ){ + for(i=0; inCol; i++){ + nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); + } } - pNew->pEList = pEList; - pNew->pSrc = pSrc; - pNew->pWhere = pWhere; - pNew->pGroupBy = pGroupBy; - pNew->pHaving = pHaving; - pNew->pOrderBy = pOrderBy; - pNew->selFlags = isDistinct ? SF_Distinct : 0; - pNew->op = TK_SELECT; - pNew->pLimit = pLimit; - pNew->pOffset = pOffset; - assert( pOffset==0 || pLimit!=0 ); - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; - if( db->mallocFailed ) { - clearSelect(db, pNew); - if( pNew!=&standin ) sqlite3DbFree(db, pNew); - pNew = 0; + if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ + sqlite3ErrorMsg(pParse, + "table %S has %d columns but %d values were supplied", + pTabList, 0, pTab->nCol-nHidden, nColumn); + goto insert_cleanup; } - return pNew; -} - -/* -** Delete the given Select structure and all of its substructures. -*/ -SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ - if( p ){ - clearSelect(db, p); - sqlite3DbFree(db, p); + if( pColumn!=0 && nColumn!=pColumn->nId ){ + sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); + goto insert_cleanup; } -} -/* -** Given 1 to 3 identifiers preceeding the JOIN keyword, determine the -** type of join. Return an integer constant that expresses that type -** in terms of the following bit values: -** -** JT_INNER -** JT_CROSS -** JT_OUTER -** JT_NATURAL -** JT_LEFT -** JT_RIGHT -** -** A full outer join is the combination of JT_LEFT and JT_RIGHT. -** -** If an illegal or unsupported join type is seen, then still return -** a join type, but put an error in the pParse structure. -*/ -SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ - int jointype = 0; - Token *apAll[3]; - Token *p; - /* 0123456789 123456789 123456789 123 */ - static const char zKeyText[] = "naturaleftouterightfullinnercross"; - static const struct { - u8 i; /* Beginning of keyword text in zKeyText[] */ - u8 nChar; /* Length of the keyword in characters */ - u8 code; /* Join type mask */ - } aKeyword[] = { - /* natural */ { 0, 7, JT_NATURAL }, - /* left */ { 6, 4, JT_LEFT|JT_OUTER }, - /* outer */ { 10, 5, JT_OUTER }, - /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, - /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, - /* inner */ { 23, 5, JT_INNER }, - /* cross */ { 28, 5, JT_INNER|JT_CROSS }, - }; - int i, j; - apAll[0] = pA; - apAll[1] = pB; - apAll[2] = pC; - for(i=0; i<3 && apAll[i]; i++){ - p = apAll[i]; - for(j=0; jn==aKeyword[j].nChar - && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ - jointype |= aKeyword[j].code; - break; - } + /* If the INSERT statement included an IDLIST term, then make sure + ** all elements of the IDLIST really are columns of the table and + ** remember the column indices. + ** + ** If the table has an INTEGER PRIMARY KEY column and that column + ** is named in the IDLIST, then record in the keyColumn variable + ** the index into IDLIST of the primary key column. keyColumn is + ** the index of the primary key as it appears in IDLIST, not as + ** is appears in the original table. (The index of the primary + ** key in the original table is pTab->iPKey.) + */ + if( pColumn ){ + for(i=0; inId; i++){ + pColumn->a[i].idx = -1; } - testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); - if( j>=ArraySize(aKeyword) ){ - jointype |= JT_ERROR; - break; + for(i=0; inId; i++){ + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ + pColumn->a[i].idx = j; + if( j==pTab->iPKey ){ + keyColumn = i; + } + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pColumn->a[i].zName) ){ + keyColumn = i; + }else{ + sqlite3ErrorMsg(pParse, "table %S has no column named %s", + pTabList, 0, pColumn->a[i].zName); + pParse->checkSchema = 1; + goto insert_cleanup; + } + } } } - if( - (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || - (jointype & JT_ERROR)!=0 - ){ - const char *zSp = " "; - assert( pB!=0 ); - if( pC==0 ){ zSp++; } - sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " - "%T %T%s%T", pA, pB, zSp, pC); - jointype = JT_INNER; - }else if( (jointype & JT_OUTER)!=0 - && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ - sqlite3ErrorMsg(pParse, - "RIGHT and FULL OUTER JOINs are not currently supported"); - jointype = JT_INNER; - } - return jointype; -} -/* -** Return the index of a column in a table. Return -1 if the column -** is not contained in the table. -*/ -static int columnIndex(Table *pTab, const char *zCol){ - int i; - for(i=0; inCol; i++){ - if( sqlite3StrICmp(pTab->aCol[i].zName, zCol)==0 ) return i; + /* If there is no IDLIST term but the table has an integer primary + ** key, the set the keyColumn variable to the primary key column index + ** in the original table definition. + */ + if( pColumn==0 && nColumn>0 ){ + keyColumn = pTab->iPKey; + } + + /* Initialize the count of rows to be inserted + */ + if( db->flags & SQLITE_CountRows ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); } - return -1; -} -/* -** Search the first N tables in pSrc, from left to right, looking for a -** table that has a column named zCol. -** -** When found, set *piTab and *piCol to the table index and column index -** of the matching column and return TRUE. -** -** If not found, return FALSE. -*/ -static int tableAndColumnIndex( - SrcList *pSrc, /* Array of tables to search */ - int N, /* Number of tables in pSrc->a[] to search */ - const char *zCol, /* Name of the column we are looking for */ - int *piTab, /* Write index of pSrc->a[] here */ - int *piCol /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ -){ - int i; /* For looping over tables in pSrc */ - int iCol; /* Index of column matching zCol */ + /* If this is not a view, open the table and and all indices */ + if( !isView ){ + int nIdx; - assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ - for(i=0; ia[i].pTab, zCol); - if( iCol>=0 ){ - if( piTab ){ - *piTab = i; - *piCol = iCol; - } - return 1; + baseCur = pParse->nTab; + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, baseCur, OP_OpenWrite); + aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); + if( aRegIdx==0 ){ + goto insert_cleanup; + } + for(i=0; inMem; } } - return 0; -} -/* -** This function is used to add terms implied by JOIN syntax to the -** WHERE clause expression of a SELECT statement. The new term, which -** is ANDed with the existing WHERE clause, is of the form: -** -** (tab1.col1 = tab2.col2) -** -** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the -** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is -** column iColRight of tab2. -*/ -static void addWhereTerm( - Parse *pParse, /* Parsing context */ - SrcList *pSrc, /* List of tables in FROM clause */ - int iLeft, /* Index of first table to join in pSrc */ - int iColLeft, /* Index of column in first table */ - int iRight, /* Index of second table in pSrc */ - int iColRight, /* Index of column in second table */ - int isOuterJoin, /* True if this is an OUTER join */ - Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ -){ - sqlite3 *db = pParse->db; - Expr *pE1; - Expr *pE2; - Expr *pEq; - - assert( iLeftnSrc>iRight ); - assert( pSrc->a[iLeft].pTab ); - assert( pSrc->a[iRight].pTab ); - - pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); - pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); - - pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); - if( pEq && isOuterJoin ){ - ExprSetProperty(pEq, EP_FromJoin); - assert( !ExprHasAnyProperty(pEq, EP_TokenOnly|EP_Reduced) ); - ExprSetIrreducible(pEq); - pEq->iRightJoinTable = (i16)pE2->iTable; + /* This is the top of the main insertion loop */ + if( useTempTable ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 4): + ** + ** rewind temp table + ** C: loop over rows of intermediate table + ** transfer values form intermediate table into
    + ** end loop + ** D: ... + */ + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); + addrCont = sqlite3VdbeCurrentAddr(v); + }else if( pSelect ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 3): + ** + ** C: yield X + ** if EOF goto D + ** insert the select result into
    from R..R+n + ** goto C + ** D: ... + */ + addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); + addrInsTop = sqlite3VdbeAddOp1(v, OP_If, regEof); } - *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); -} - -/* -** Set the EP_FromJoin property on all terms of the given expression. -** And set the Expr.iRightJoinTable to iTable for every term in the -** expression. -** -** The EP_FromJoin property is used on terms of an expression to tell -** the LEFT OUTER JOIN processing logic that this term is part of the -** join restriction specified in the ON or USING clause and not a part -** of the more general WHERE clause. These terms are moved over to the -** WHERE clause during join processing but we need to remember that they -** originated in the ON or USING clause. -** -** The Expr.iRightJoinTable tells the WHERE clause processing that the -** expression depends on table iRightJoinTable even if that table is not -** explicitly mentioned in the expression. That information is needed -** for cases like this: -** -** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5 -** -** The where clause needs to defer the handling of the t1.x=5 -** term until after the t2 loop of the join. In that way, a -** NULL t2 row will be inserted whenever t1.x!=5. If we do not -** defer the handling of t1.x=5, it will be processed immediately -** after the t1 loop and rows with t1.x!=5 will never appear in -** the output, which is incorrect. -*/ -static void setJoinExpr(Expr *p, int iTable){ - while( p ){ - ExprSetProperty(p, EP_FromJoin); - assert( !ExprHasAnyProperty(p, EP_TokenOnly|EP_Reduced) ); - ExprSetIrreducible(p); - p->iRightJoinTable = (i16)iTable; - setJoinExpr(p->pLeft, iTable); - p = p->pRight; - } -} - -/* -** This routine processes the join information for a SELECT statement. -** ON and USING clauses are converted into extra terms of the WHERE clause. -** NATURAL joins also create extra WHERE clause terms. -** -** The terms of a FROM clause are contained in the Select.pSrc structure. -** The left most table is the first entry in Select.pSrc. The right-most -** table is the last entry. The join operator is held in the entry to -** the left. Thus entry 0 contains the join operator for the join between -** entries 0 and 1. Any ON or USING clauses associated with the join are -** also attached to the left entry. -** -** This routine returns the number of errors encountered. -*/ -static int sqliteProcessJoin(Parse *pParse, Select *p){ - SrcList *pSrc; /* All tables in the FROM clause */ - int i, j; /* Loop counters */ - struct SrcList_item *pLeft; /* Left table being joined */ - struct SrcList_item *pRight; /* Right table being joined */ - pSrc = p->pSrc; - pLeft = &pSrc->a[0]; - pRight = &pLeft[1]; - for(i=0; inSrc-1; i++, pRight++, pLeft++){ - Table *pLeftTab = pLeft->pTab; - Table *pRightTab = pRight->pTab; - int isOuter; + /* Allocate registers for holding the rowid of the new row, + ** the content of the new row, and the assemblied row record. + */ + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; - if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->jointype & JT_OUTER)!=0; + /* Run the BEFORE and INSTEAD OF triggers, if there are any + */ + endOfLoop = sqlite3VdbeMakeLabel(v); + if( tmask & TRIGGER_BEFORE ){ + int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); - /* When the NATURAL keyword is present, add WHERE clause terms for - ** every column that the two tables have in common. + /* build the NEW.* reference row. Note that if there is an INTEGER + ** PRIMARY KEY into which a NULL is being inserted, that NULL will be + ** translated into a unique ID for the row. But on a BEFORE trigger, + ** we do not know what the unique ID will be (because the insert has + ** not happened yet) so we substitute a rowid of -1 */ - if( pRight->jointype & JT_NATURAL ){ - if( pRight->pOn || pRight->pUsing ){ - sqlite3ErrorMsg(pParse, "a NATURAL join may not have " - "an ON or USING clause", 0); - return 1; + if( keyColumn<0 ){ + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + }else{ + int j1; + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regCols); + }else{ + assert( pSelect==0 ); /* Otherwise useTempTable is true */ + sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regCols); } - for(j=0; jnCol; j++){ - char *zName; /* Name of column in the right table */ - int iLeft; /* Matching left table */ - int iLeftCol; /* Matching column in the left table */ + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); + } - zName = pRightTab->aCol[j].zName; - if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) ){ - addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, - isOuter, &p->pWhere); + /* Cannot have triggers on a virtual table. If it were possible, + ** this block would have to account for hidden column. + */ + assert( !IsVirtual(pTab) ); + + /* Create the new column data + */ + for(i=0; inCol; i++){ + if( pColumn==0 ){ + j = i; + }else{ + for(j=0; jnId; j++){ + if( pColumn->a[j].idx==i ) break; } } + if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){ + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); + }else if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); + }else{ + assert( pSelect==0 ); /* Otherwise useTempTable is true */ + sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); + } } - /* Disallow both ON and USING clauses in the same join + /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, + ** do not attempt any conversions before assembling the record. + ** If this is a real table, attempt conversions as required by the + ** table column affinities. */ - if( pRight->pOn && pRight->pUsing ){ - sqlite3ErrorMsg(pParse, "cannot have both ON and USING " - "clauses in the same join"); - return 1; + if( !isView ){ + sqlite3VdbeAddOp2(v, OP_Affinity, regCols+1, pTab->nCol); + sqlite3TableAffinityStr(v, pTab); } - /* Add the ON clause to the end of the WHERE clause, connected by - ** an AND operator. - */ - if( pRight->pOn ){ - if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); - p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); - pRight->pOn = 0; + /* Fire BEFORE or INSTEAD OF triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, + pTab, regCols-pTab->nCol-1, onError, endOfLoop); + + sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); + } + + /* Push the record number for the new entry onto the stack. The + ** record number is a randomly generate integer created by NewRowid + ** except when the table has an INTEGER PRIMARY KEY column, in which + ** case the record number is the same as that column. + */ + if( !isView ){ + if( IsVirtual(pTab) ){ + /* The row that the VUpdate opcode will delete: none */ + sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); + } + if( keyColumn>=0 ){ + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regRowid); + }else if( pSelect ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+keyColumn, regRowid); + }else{ + VdbeOp *pOp; + sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regRowid); + pOp = sqlite3VdbeGetOp(v, -1); + if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ + appendFlag = 1; + pOp->opcode = OP_NewRowid; + pOp->p1 = baseCur; + pOp->p2 = regRowid; + pOp->p3 = regAutoinc; + } + } + /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid + ** to generate a unique primary key value. + */ + if( !appendFlag ){ + int j1; + if( !IsVirtual(pTab) ){ + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); + sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); + sqlite3VdbeJumpHere(v, j1); + }else{ + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); + } + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); + } + }else if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); + }else{ + sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); + appendFlag = 1; } + autoIncStep(pParse, regAutoinc, regRowid); - /* Create extra terms on the WHERE clause for each column named - ** in the USING clause. Example: If the two tables to be joined are - ** A and B and the USING clause names X, Y, and Z, then add this - ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z - ** Report an error if any column mentioned in the USING clause is - ** not contained in both tables to be joined. + /* Push onto the stack, data for all columns of the new entry, beginning + ** with the first column. */ - if( pRight->pUsing ){ - IdList *pList = pRight->pUsing; - for(j=0; jnId; j++){ - char *zName; /* Name of the term in the USING clause */ - int iLeft; /* Table on the left with matching column name */ - int iLeftCol; /* Column number of matching column on the left */ - int iRightCol; /* Column number of matching column on the right */ - - zName = pList->a[j].zName; - iRightCol = columnIndex(pRightTab, zName); - if( iRightCol<0 - || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) - ){ - sqlite3ErrorMsg(pParse, "cannot join using column %s - column " - "not present in both tables", zName); - return 1; + nHidden = 0; + for(i=0; inCol; i++){ + int iRegStore = regRowid+1+i; + if( i==pTab->iPKey ){ + /* The value of the INTEGER PRIMARY KEY column is always a NULL. + ** Whenever this column is read, the record number will be substituted + ** in its place. So will fill this column with a NULL to avoid + ** taking up data space with information that will never be used. */ + sqlite3VdbeAddOp2(v, OP_Null, 0, iRegStore); + continue; + } + if( pColumn==0 ){ + if( IsHiddenColumn(&pTab->aCol[i]) ){ + assert( IsVirtual(pTab) ); + j = -1; + nHidden++; + }else{ + j = i - nHidden; } - addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, - isOuter, &p->pWhere); + }else{ + for(j=0; jnId; j++){ + if( pColumn->a[j].idx==i ) break; + } + } + if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, iRegStore); + }else if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); + }else if( pSelect ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); + }else{ + sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); } } - } - return 0; -} -/* -** Insert code into "v" that will push the record on the top of the -** stack into the sorter. -*/ -static void pushOntoSorter( - Parse *pParse, /* Parser context */ - ExprList *pOrderBy, /* The ORDER BY clause */ - Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ -){ - Vdbe *v = pParse->pVdbe; - int nExpr = pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); - if( pSelect->iLimit ){ - int addr1, addr2; - int iLimit; - if( pSelect->iOffset ){ - iLimit = pSelect->iOffset+1; - }else{ - iLimit = pSelect->iLimit; + /* Generate code to check constraints and generate index keys and + ** do the insertion. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); + sqlite3MayAbort(pParse); + }else +#endif + { + int isReplace; /* Set to true if constraints may cause a replace */ + sqlite3GenerateConstraintChecks(pParse, pTab, baseCur, regIns, aRegIdx, + keyColumn>=0, 0, onError, endOfLoop, &isReplace + ); + sqlite3FkCheck(pParse, pTab, 0, regIns); + sqlite3CompleteInsertion( + pParse, pTab, baseCur, regIns, aRegIdx, 0, appendFlag, isReplace==0 + ); } - addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); - sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); - addr2 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); - sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); - sqlite3VdbeJumpHere(v, addr2); } -} -/* -** Add code to implement the OFFSET -*/ -static void codeOffset( - Vdbe *v, /* Generate code into this VM */ - Select *p, /* The SELECT statement being coded */ - int iContinue /* Jump here to skip the current record */ -){ - if( p->iOffset && iContinue!=0 ){ - int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); - VdbeComment((v, "skip OFFSET records")); - sqlite3VdbeJumpHere(v, addr); + /* Update the count of rows that are inserted + */ + if( (db->flags & SQLITE_CountRows)!=0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); } -} -/* -** Add code that will check to make sure the N registers starting at iMem -** form a distinct entry. iTab is a sorting index that holds previously -** seen combinations of the N values. A new entry is made in iTab -** if the current N values are new. -** -** A jump to addrRepeat is made and the N+1 values are popped from the -** stack if the top N elements are not distinct. -*/ -static void codeDistinct( - Parse *pParse, /* Parsing and code generating context */ - int iTab, /* A sorting index used to test for distinctness */ - int addrRepeat, /* Jump to here if not distinct */ - int N, /* Number of elements */ - int iMem /* First element */ -){ - Vdbe *v; - int r1; + if( pTrigger ){ + /* Code AFTER triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, + pTab, regData-2-pTab->nCol, onError, endOfLoop); + } - v = pParse->pVdbe; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); - sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); - sqlite3ReleaseTempReg(pParse, r1); -} + /* The bottom of the main insertion loop, if the data source + ** is a SELECT statement. + */ + sqlite3VdbeResolveLabel(v, endOfLoop); + if( useTempTable ){ + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); + sqlite3VdbeJumpHere(v, addrInsTop); + sqlite3VdbeAddOp1(v, OP_Close, srcTab); + }else if( pSelect ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeJumpHere(v, addrInsTop); + } -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Generate an error message when a SELECT is used within a subexpression -** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error used to occur -** in multiple places. (The error only occurs in one place now, but we -** retain the subroutine to minimize code disruption.) -*/ -static int checkForMultiColumnSelectError( - Parse *pParse, /* Parse context. */ - SelectDest *pDest, /* Destination of SELECT results */ - int nExpr /* Number of result columns returned by SELECT */ -){ - int eDest = pDest->eDest; - if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ - sqlite3ErrorMsg(pParse, "only a single result allowed for " - "a SELECT that is part of an expression"); - return 1; - }else{ - return 0; + if( !IsVirtual(pTab) && !isView ){ + /* Close all tables opened */ + sqlite3VdbeAddOp1(v, OP_Close, baseCur); + for(idx=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, idx++){ + sqlite3VdbeAddOp1(v, OP_Close, idx+baseCur); + } + } + +insert_end: + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows inserted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); } + +insert_cleanup: + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aRegIdx); } -#endif -/* -** This routine generates the code for the inside of the inner loop -** of a SELECT. -** -** If srcTab and nColumn are both zero, then the pEList expressions -** are evaluated in order to get the data for this row. If nColumn>0 -** then data is pulled from srcTab and pEList is used only to get the -** datatypes for each column. +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** thely may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif +#ifdef tmask + #undef tmask +#endif + + +/* +** Generate code to do constraint checks prior to an INSERT or an UPDATE. +** +** The input is a range of consecutive registers as follows: +** +** 1. The rowid of the row after the update. +** +** 2. The data in the first column of the entry after the update. +** +** i. Data from middle columns... +** +** N. The data in the last column of the entry after the update. +** +** The regRowid parameter is the index of the register containing (1). +** +** If isUpdate is true and rowidChng is non-zero, then rowidChng contains +** the address of a register containing the rowid before the update takes +** place. isUpdate is true for UPDATEs and false for INSERTs. If isUpdate +** is false, indicating an INSERT statement, then a non-zero rowidChng +** indicates that the rowid was explicitly specified as part of the +** INSERT statement. If rowidChng is false, it means that the rowid is +** computed automatically in an insert or that the rowid value is not +** modified by an update. +** +** The code generated by this routine store new index entries into +** registers identified by aRegIdx[]. No index entry is created for +** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is +** the same as the order of indices on the linked list of indices +** attached to the table. +** +** This routine also generates code to check constraints. NOT NULL, +** CHECK, and UNIQUE constraints are all checked. If a constraint fails, +** then the appropriate action is performed. There are five possible +** actions: ROLLBACK, ABORT, FAIL, REPLACE, and IGNORE. +** +** Constraint type Action What Happens +** --------------- ---------- ---------------------------------------- +** any ROLLBACK The current transaction is rolled back and +** sqlite3_exec() returns immediately with a +** return code of SQLITE_CONSTRAINT. +** +** any ABORT Back out changes from the current command +** only (do not do a complete rollback) then +** cause sqlite3_exec() to return immediately +** with SQLITE_CONSTRAINT. +** +** any FAIL Sqlite_exec() returns immediately with a +** return code of SQLITE_CONSTRAINT. The +** transaction is not rolled back and any +** prior changes are retained. +** +** any IGNORE The record number and data is popped from +** the stack and there is an immediate jump +** to label ignoreDest. +** +** NOT NULL REPLACE The NULL value is replace by the default +** value for that column. If the default value +** is NULL, the action is the same as ABORT. +** +** UNIQUE REPLACE The other row that conflicts with the row +** being inserted is removed. +** +** CHECK REPLACE Illegal. The results in an exception. +** +** Which action to take is determined by the overrideError parameter. +** Or if overrideError==OE_Default, then the pParse->onError parameter +** is used. Or if pParse->onError==OE_Default then the onError value +** for the constraint is used. +** +** The calling routine must open a read/write cursor for pTab with +** cursor number "baseCur". All indices of pTab must also have open +** read/write cursors with cursor number baseCur+i for the i-th cursor. +** Except, if there is no possibility of a REPLACE action then +** cursors do not need to be open for indices where aRegIdx[i]==0. */ -static void selectInnerLoop( - Parse *pParse, /* The parser context */ - Select *p, /* The complete select statement being coded */ - ExprList *pEList, /* List of values being extracted */ - int srcTab, /* Pull data from this table */ - int nColumn, /* Number of columns in the source table */ - ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int distinct, /* If >=0, make sure results are distinct */ - SelectDest *pDest, /* How to dispose of the results */ - int iContinue, /* Jump here to continue with next row */ - int iBreak /* Jump here to break out of the inner loop */ +SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( + Parse *pParse, /* The parser context */ + Table *pTab, /* the table into which we are inserting */ + int baseCur, /* Index of a read/write cursor pointing at pTab */ + int regRowid, /* Index of the range of input registers */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ + int rowidChng, /* True if the rowid might collide with existing entry */ + int isUpdate, /* True for UPDATE, False for INSERT */ + int overrideError, /* Override onError to this if not OE_Default */ + int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ + int *pbMayReplace /* OUT: Set to true if constraint may cause a replace */ ){ - Vdbe *v = pParse->pVdbe; - int i; - int hasDistinct; /* True if the DISTINCT keyword is present */ - int regResult; /* Start of memory holding result set */ - int eDest = pDest->eDest; /* How to dispose of results */ - int iParm = pDest->iParm; /* First argument to disposal method */ - int nResultCol; /* Number of result columns */ + int i; /* loop counter */ + Vdbe *v; /* VDBE under constrution */ + int nCol; /* Number of columns */ + int onError; /* Conflict resolution strategy */ + int j1; /* Addresss of jump instruction */ + int j2 = 0, j3; /* Addresses of jump instructions */ + int regData; /* Register containing first data column */ + int iCur; /* Table cursor number */ + Index *pIdx; /* Pointer to one of the indices */ + int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ + int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; - assert( v ); - if( NEVER(v==0) ) return; - assert( pEList!=0 ); - hasDistinct = distinct>=0; - if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue); - } + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + nCol = pTab->nCol; + regData = regRowid + 1; - /* Pull the requested columns. + /* Test all NOT NULL constraints. */ - if( nColumn>0 ){ - nResultCol = nColumn; - }else{ - nResultCol = pEList->nExpr; - } - if( pDest->iMem==0 ){ - pDest->iMem = pParse->nMem+1; - pDest->nMem = nResultCol; - pParse->nMem += nResultCol; - }else{ - assert( pDest->nMem==nResultCol ); - } - regResult = pDest->iMem; - if( nColumn>0 ){ - for(i=0; iiPKey ){ + continue; + } + onError = pTab->aCol[i].notNull; + if( onError==OE_None ) continue; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + if( onError==OE_Replace && pTab->aCol[i].pDflt==0 ){ + onError = OE_Abort; + } + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + switch( onError ){ + case OE_Abort: + sqlite3MayAbort(pParse); + case OE_Rollback: + case OE_Fail: { + char *zMsg; + sqlite3VdbeAddOp3(v, OP_HaltIfNull, + SQLITE_CONSTRAINT, onError, regData+i); + zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", + pTab->zName, pTab->aCol[i].zName); + sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); + break; + } + case OE_Ignore: { + sqlite3VdbeAddOp2(v, OP_IsNull, regData+i, ignoreDest); + break; + } + default: { + assert( onError==OE_Replace ); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regData+i); + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regData+i); + sqlite3VdbeJumpHere(v, j1); + break; + } } - }else if( eDest!=SRT_Exists ){ - /* If the destination is an EXISTS(...) expression, the actual - ** values returned by the SELECT are not required. - */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pEList, regResult, eDest==SRT_Output); } - nColumn = nResultCol; - /* If the DISTINCT keyword was present on the SELECT statement - ** and this row has been seen before, then do not make this row - ** part of the result. + /* Test all CHECK constraints */ - if( hasDistinct ){ - assert( pEList!=0 ); - assert( pEList->nExpr==nColumn ); - codeDistinct(pParse, distinct, iContinue, nColumn, regResult); - if( pOrderBy==0 ){ - codeOffset(v, p, iContinue); +#ifndef SQLITE_OMIT_CHECK + if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ + int allOk = sqlite3VdbeMakeLabel(v); + pParse->ckBase = regData; + sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); + onError = overrideError!=OE_Default ? overrideError : OE_Abort; + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + sqlite3HaltConstraint(pParse, onError, 0, 0); } + sqlite3VdbeResolveLabel(v, allOk); } +#endif /* !defined(SQLITE_OMIT_CHECK) */ - switch( eDest ){ - /* In this mode, write each query result to the key of the temporary - ** table iParm. - */ -#ifndef SQLITE_OMIT_COMPOUND_SELECT - case SRT_Union: { - int r1; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); - sqlite3ReleaseTempReg(pParse, r1); - break; + /* If we have an INTEGER PRIMARY KEY, make sure the primary key + ** of the new record does not previously exist. Except, if this + ** is an UPDATE and the primary key is not changing, that is OK. + */ + if( rowidChng ){ + onError = pTab->keyConf; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; } - - /* Construct a record from the query result, but instead of - ** saving that record, use it as a key to delete elements from - ** the temporary table iParm. - */ - case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); - break; + + if( isUpdate ){ + j2 = sqlite3VdbeAddOp3(v, OP_Eq, regRowid, 0, rowidChng); } -#endif - - /* Store the result as data using a unique key. - */ - case SRT_Table: - case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); - testcase( eDest==SRT_Table ); - testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, r1); - }else{ - int r2 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3ReleaseTempReg(pParse, r2); + j3 = sqlite3VdbeAddOp3(v, OP_NotExists, baseCur, 0, regRowid); + switch( onError ){ + default: { + onError = OE_Abort; + /* Fall thru into the next case */ } - sqlite3ReleaseTempReg(pParse, r1); - break; - } - -#ifndef SQLITE_OMIT_SUBQUERY - /* If we are creating a set for an "expr IN (SELECT ...)" construct, - ** then there should be a single item on the stack. Write this - ** item into the set table with bogus data. - */ - case SRT_Set: { - assert( nColumn==1 ); - p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affinity); - if( pOrderBy ){ - /* At first glance you would think we could optimize out the - ** ORDER BY in this case since the order of entries in the set - ** does not matter. But there might be a LIMIT clause, in which - ** case the order does matter */ - pushOntoSorter(pParse, pOrderBy, p, regResult); - }else{ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, regResult, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); - sqlite3ReleaseTempReg(pParse, r1); + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + sqlite3HaltConstraint( + pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + break; } - break; - } - - /* If any row exist in the result set, record that fact and abort. - */ - case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm); - /* The LIMIT clause will terminate the loop for us */ - break; - } - - /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell and break out - ** of the scan loop. - */ - case SRT_Mem: { - assert( nColumn==1 ); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, regResult); - }else{ - sqlite3ExprCodeMove(pParse, regResult, iParm, 1); - /* The LIMIT clause will jump out of the loop for us */ + case OE_Replace: { + /* If there are DELETE triggers on this table and the + ** recursive-triggers flag is set, call GenerateRowDelete() to + ** remove the conflicting row from the the table. This will fire + ** the triggers and remove both the table and index b-tree entries. + ** + ** Otherwise, if there are no triggers or the recursive-triggers + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. + ** + ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, + ** also invoke MultiWrite() to indicate that this VDBE may require + ** statement rollback (if the statement is aborted after the delete + ** takes place). Earlier versions called sqlite3MultiWrite() regardless, + ** but being more selective here allows statements like: + ** + ** REPLACE INTO t(rowid) VALUES($newrowid) + ** + ** to run without a statement journal if there are no indexes on the + ** table. + */ + Trigger *pTrigger = 0; + if( pParse->db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + } + if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowDelete( + pParse, pTab, baseCur, regRowid, 0, pTrigger, OE_Replace + ); + }else if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, baseCur, 0); + } + seenReplace = 1; + break; } - break; - } -#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ - - /* Send the data to the callback function or to a subroutine. In the - ** case of a subroutine, the subroutine itself is responsible for - ** popping the data from the stack. - */ - case SRT_Coroutine: - case SRT_Output: { - testcase( eDest==SRT_Coroutine ); - testcase( eDest==SRT_Output ); - if( pOrderBy ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); - pushOntoSorter(pParse, pOrderBy, p, r1); - sqlite3ReleaseTempReg(pParse, r1); - }else if( eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); - }else{ - sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); + case OE_Ignore: { + assert( seenReplace==0 ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + break; } - break; } - -#if !defined(SQLITE_OMIT_TRIGGER) - /* Discard the results. This is used for SELECT statements inside - ** the body of a TRIGGER. The purpose of such selects is to call - ** user-defined functions that have side effects. We do not care - ** about the actual results of the select. - */ - default: { - assert( eDest==SRT_Discard ); - break; + sqlite3VdbeJumpHere(v, j3); + if( isUpdate ){ + sqlite3VdbeJumpHere(v, j2); } -#endif } - /* Jump to the end of the loop if the LIMIT is reached. Except, if - ** there is a sorter, in which case the sorter has already limited - ** the output for us. + /* Test all UNIQUE constraints by creating entries for each UNIQUE + ** index and making sure that duplicate entries do not already exist. + ** Add the new records to the indices as we go. */ - if( pOrderBy==0 && p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); - } -} + for(iCur=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, iCur++){ + int regIdx; + int regR; -/* -** Given an expression list, generate a KeyInfo structure that records -** the collating sequence for each expression in that expression list. -** -** If the ExprList is an ORDER BY or GROUP BY clause then the resulting -** KeyInfo structure is appropriate for initializing a virtual index to -** implement that clause. If the ExprList is the result set of a SELECT -** then the KeyInfo structure is appropriate for initializing a virtual -** index to implement a DISTINCT test. -** -** Space to hold the KeyInfo structure is obtain from malloc. The calling -** function is responsible for seeing that this structure is eventually -** freed. Add the KeyInfo structure to the P4 field of an opcode using -** P4_KEYINFO_HANDOFF is the usual way of dealing with this. -*/ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ - sqlite3 *db = pParse->db; - int nExpr; - KeyInfo *pInfo; - struct ExprList_item *pItem; - int i; + if( aRegIdx[iCur]==0 ) continue; /* Skip unused indices */ - nExpr = pList->nExpr; - pInfo = sqlite3DbMallocZero(db, sizeof(*pInfo) + nExpr*(sizeof(CollSeq*)+1) ); - if( pInfo ){ - pInfo->aSortOrder = (u8*)&pInfo->aColl[nExpr]; - pInfo->nField = (u16)nExpr; - pInfo->enc = ENC(db); - pInfo->db = db; - for(i=0, pItem=pList->a; ipExpr); - if( !pColl ){ - pColl = db->pDfltColl; + /* Create a key for accessing the index entry */ + regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn+1); + for(i=0; inColumn; i++){ + int idx = pIdx->aiColumn[i]; + if( idx==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, regData+idx, regIdx+i); } - pInfo->aColl[i] = pColl; - pInfo->aSortOrder[i] = pItem->sortOrder; } - } - return pInfo; -} - -#ifndef SQLITE_OMIT_COMPOUND_SELECT -/* -** Name of the connection operator, used for error messages. -*/ -static const char *selectOpName(int id){ - char *z; - switch( id ){ - case TK_ALL: z = "UNION ALL"; break; - case TK_INTERSECT: z = "INTERSECT"; break; - case TK_EXCEPT: z = "EXCEPT"; break; - default: z = "UNION"; break; - } - return z; -} -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn+1, aRegIdx[iCur]); + sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), P4_TRANSIENT); + sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn+1); -#ifndef SQLITE_OMIT_EXPLAIN -/* -** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function -** is a no-op. Otherwise, it adds a single row of output to the EQP result, -** where the caption is of the form: -** -** "USE TEMP B-TREE FOR xxx" -** -** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which -** is determined by the zUsage argument. -*/ -static void explainTempTable(Parse *pParse, const char *zUsage){ - if( pParse->explain==2 ){ - Vdbe *v = pParse->pVdbe; - char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage); - sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); - } -} - -/* -** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function -** is a no-op. Otherwise, it adds a single row of output to the EQP result, -** where the caption is of one of the two forms: -** -** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)" -** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)" -** -** where iSub1 and iSub2 are the integers passed as the corresponding -** function parameters, and op is the text representation of the parameter -** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT, -** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is -** false, or the second form if it is true. -*/ -static void explainComposite( - Parse *pParse, /* Parse context */ - int op, /* One of TK_UNION, TK_EXCEPT etc. */ - int iSub1, /* Subquery id 1 */ - int iSub2, /* Subquery id 2 */ - int bUseTmp /* True if a temp table was used */ -){ - assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL ); - if( pParse->explain==2 ){ - Vdbe *v = pParse->pVdbe; - char *zMsg = sqlite3MPrintf( - pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2, - bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op) - ); - sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); - } -} - -/* -** Assign expression b to lvalue a. A second, no-op, version of this macro -** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code -** in sqlite3Select() to assign values to structure member variables that -** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the -** code with #ifndef directives. -*/ -# define explainSetInteger(a, b) a = b - -#else -/* No-op versions of the explainXXX() functions and macros. */ -# define explainTempTable(y,z) -# define explainComposite(v,w,x,y,z) -# define explainSetInteger(y,z) -#endif - -/* -** If the inner loop was generated using a non-null pOrderBy argument, -** then the results were placed in a sorter. After the loop is terminated -** we need to run the sorter and output the results. The following -** routine generates the code needed to do that. -*/ -static void generateSortTail( - Parse *pParse, /* Parsing context */ - Select *p, /* The SELECT statement */ - Vdbe *v, /* Generate code into this VDBE */ - int nColumn, /* Number of columns of data */ - SelectDest *pDest /* Write the sorted results here */ -){ - int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ - int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ - int addr; - int iTab; - int pseudoTab = 0; - ExprList *pOrderBy = p->pOrderBy; - - int eDest = pDest->eDest; - int iParm = pDest->iParm; - - int regRow; - int regRowid; - - iTab = pOrderBy->iECursor; - regRow = sqlite3GetTempReg(pParse); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); - regRowid = 0; - }else{ - regRowid = sqlite3GetTempReg(pParse); - } - addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow); - switch( eDest ){ - case SRT_Table: - case SRT_EphemTab: { - testcase( eDest==SRT_Table ); - testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case SRT_Set: { - assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, regRow, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); - break; + /* Find out what action to take in case there is an indexing conflict */ + onError = pIdx->onError; + if( onError==OE_None ){ + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); + continue; /* pIdx is not a UNIQUE index */ } - case SRT_Mem: { - assert( nColumn==1 ); - sqlite3ExprCodeMove(pParse, regRow, iParm, 1); - /* The LIMIT clause will terminate the loop for us */ - break; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; } -#endif - default: { - int i; - assert( eDest==SRT_Output || eDest==SRT_Coroutine ); - testcase( eDest==SRT_Output ); - testcase( eDest==SRT_Coroutine ); - for(i=0; iiMem+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iMem+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } - if( eDest==SRT_Output ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iMem, nColumn); - sqlite3ExprCacheAffinityChange(pParse, pDest->iMem, nColumn); - }else{ - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); - } - break; + if( seenReplace ){ + if( onError==OE_Ignore ) onError = OE_Replace; + else if( onError==OE_Fail ) onError = OE_Abort; } - } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - - /* The bottom of the loop - */ - sqlite3VdbeResolveLabel(v, addrContinue); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); - sqlite3VdbeResolveLabel(v, addrBreak); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); - } -} + + /* Check to see if the new index entry will be unique */ + regR = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_SCopy, regOldRowid, regR); + j3 = sqlite3VdbeAddOp4(v, OP_IsUnique, baseCur+iCur+1, 0, + regR, SQLITE_INT_TO_PTR(regIdx), + P4_INT32); + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); -/* -** Return a pointer to a string containing the 'declaration type' of the -** expression pExpr. The string may be treated as static by the caller. -** -** The declaration type is the exact datatype definition extracted from the -** original CREATE TABLE statement if the expression is a column. The -** declaration type for a ROWID field is INTEGER. Exactly when an expression -** is considered a column can be complex in the presence of subqueries. The -** result-set expression in all of the following SELECT statements is -** considered a column by this function. -** -** SELECT col FROM tbl; -** SELECT (SELECT col FROM tbl; -** SELECT (SELECT col FROM tbl); -** SELECT abc FROM (SELECT col AS abc FROM tbl); -** -** The declaration type for any expression other than a column is NULL. -*/ -static const char *columnType( - NameContext *pNC, - Expr *pExpr, - const char **pzOriginDb, - const char **pzOriginTab, - const char **pzOriginCol -){ - char const *zType = 0; - char const *zOriginDb = 0; - char const *zOriginTab = 0; - char const *zOriginCol = 0; - int j; - if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; + /* Generate code that executes if the new index entry is not unique */ + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + switch( onError ){ + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + int j; + StrAccum errMsg; + const char *zSep; + char *zErr; - switch( pExpr->op ){ - case TK_AGG_COLUMN: - case TK_COLUMN: { - /* The expression is a column. Locate the table the column is being - ** extracted from in NameContext.pSrcList. This table may be real - ** database table or a subquery. - */ - Table *pTab = 0; /* Table structure column is extracted from */ - Select *pS = 0; /* Select the column is extracted from */ - int iCol = pExpr->iColumn; /* Index of column in pTab */ - testcase( pExpr->op==TK_AGG_COLUMN ); - testcase( pExpr->op==TK_COLUMN ); - while( pNC && !pTab ){ - SrcList *pTabList = pNC->pSrcList; - for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); - if( jnSrc ){ - pTab = pTabList->a[j].pTab; - pS = pTabList->a[j].pSelect; - }else{ - pNC = pNC->pNext; + sqlite3StrAccumInit(&errMsg, 0, 0, 200); + errMsg.db = pParse->db; + zSep = pIdx->nColumn>1 ? "columns " : "column "; + for(j=0; jnColumn; j++){ + char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + sqlite3StrAccumAppend(&errMsg, zSep, -1); + zSep = ", "; + sqlite3StrAccumAppend(&errMsg, zCol, -1); } + sqlite3StrAccumAppend(&errMsg, + pIdx->nColumn>1 ? " are not unique" : " is not unique", -1); + zErr = sqlite3StrAccumFinish(&errMsg); + sqlite3HaltConstraint(pParse, onError, zErr, 0); + sqlite3DbFree(errMsg.db, zErr); + break; } - - if( pTab==0 ){ - /* At one time, code such as "SELECT new.x" within a trigger would - ** cause this condition to run. Since then, we have restructured how - ** trigger code is generated and so this condition is no longer - ** possible. However, it can still be true for statements like - ** the following: - ** - ** CREATE TABLE t1(col INTEGER); - ** SELECT (SELECT t1.col) FROM FROM t1; - ** - ** when columnType() is called on the expression "t1.col" in the - ** sub-select. In this case, set the column type to NULL, even - ** though it should really be "INTEGER". - ** - ** This is not a problem, as the column type of "t1.col" is never - ** used. When columnType() is called on the expression - ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT - ** branch below. */ + case OE_Ignore: { + assert( seenReplace==0 ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } - - assert( pTab && pExpr->pTab==pTab ); - if( pS ){ - /* The "table" is actually a sub-select or a view in the FROM clause - ** of the SELECT statement. Return the declaration type and origin - ** data for the result-set column of the sub-select. - */ - if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ - /* If iCol is less than zero, then the expression requests the - ** rowid of the sub-select or view. This expression is legal (see - ** test case misc2.2.2) - it always evaluates to NULL. - */ - NameContext sNC; - Expr *p = pS->pEList->a[iCol].pExpr; - sNC.pSrcList = pS->pSrc; - sNC.pNext = pNC; - sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); - } - }else if( ALWAYS(pTab->pSchema) ){ - /* A real table */ - assert( !pS ); - if( iCol<0 ) iCol = pTab->iPKey; - assert( iCol==-1 || (iCol>=0 && iColnCol) ); - if( iCol<0 ){ - zType = "INTEGER"; - zOriginCol = "rowid"; - }else{ - zType = pTab->aCol[iCol].zType; - zOriginCol = pTab->aCol[iCol].zName; - } - zOriginTab = pTab->zName; - if( pNC->pParse ){ - int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOriginDb = pNC->pParse->db->aDb[iDb].zName; + default: { + Trigger *pTrigger = 0; + assert( onError==OE_Replace ); + sqlite3MultiWrite(pParse); + if( pParse->db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } + sqlite3GenerateRowDelete( + pParse, pTab, baseCur, regR, 0, pTrigger, OE_Replace + ); + seenReplace = 1; + break; } - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_SELECT: { - /* The expression is a sub-select. Return the declaration type and - ** origin info for the single column in the result set of the SELECT - ** statement. - */ - NameContext sNC; - Select *pS = pExpr->x.pSelect; - Expr *p = pS->pEList->a[0].pExpr; - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); - sNC.pSrcList = pS->pSrc; - sNC.pNext = pNC; - sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); - break; } -#endif + sqlite3VdbeJumpHere(v, j3); + sqlite3ReleaseTempReg(pParse, regR); } - if( pzOriginDb ){ - assert( pzOriginTab && pzOriginCol ); - *pzOriginDb = zOriginDb; - *pzOriginTab = zOriginTab; - *pzOriginCol = zOriginCol; + if( pbMayReplace ){ + *pbMayReplace = seenReplace; } - return zType; } /* -** Generate code that will tell the VDBE the declaration types of columns -** in the result set. +** This routine generates code to finish the INSERT or UPDATE operation +** that was started by a prior call to sqlite3GenerateConstraintChecks. +** A consecutive range of registers starting at regRowid contains the +** rowid and the content to be inserted. +** +** The arguments to this routine should be the same as the first six +** arguments to sqlite3GenerateConstraintChecks. */ -static void generateColumnTypes( - Parse *pParse, /* Parser context */ - SrcList *pTabList, /* List of tables */ - ExprList *pEList /* Expressions defining the result set */ +SQLITE_PRIVATE void sqlite3CompleteInsertion( + Parse *pParse, /* The parser context */ + Table *pTab, /* the table into which we are inserting */ + int baseCur, /* Index of a read/write cursor pointing at pTab */ + int regRowid, /* Range of content */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ + int isUpdate, /* True for UPDATE, False for INSERT */ + int appendBias, /* True if this is likely to be an append */ + int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ ){ -#ifndef SQLITE_OMIT_DECLTYPE - Vdbe *v = pParse->pVdbe; int i; - NameContext sNC; - sNC.pSrcList = pTabList; - sNC.pParse = pParse; - for(i=0; inExpr; i++){ - Expr *p = pEList->a[i].pExpr; - const char *zType; -#ifdef SQLITE_ENABLE_COLUMN_METADATA - const char *zOrigDb = 0; - const char *zOrigTab = 0; - const char *zOrigCol = 0; - zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + Vdbe *v; + int nIdx; + Index *pIdx; + u8 pik_flags; + int regData; + int regRec; - /* The vdbe must make its own copy of the column-type and other - ** column specific strings, in case the schema is reset before this - ** virtual machine is deleted. - */ - sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT); - sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); - sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); -#else - zType = columnType(&sNC, p, 0, 0, 0); -#endif - sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} + for(i=nIdx-1; i>=0; i--){ + if( aRegIdx[i]==0 ) continue; + sqlite3VdbeAddOp2(v, OP_IdxInsert, baseCur+i+1, aRegIdx[i]); + if( useSeekResult ){ + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } } -#endif /* SQLITE_OMIT_DECLTYPE */ + regData = regRowid + 1; + regRec = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); + sqlite3TableAffinityStr(v, pTab); + sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); + if( pParse->nested ){ + pik_flags = 0; + }else{ + pik_flags = OPFLAG_NCHANGE; + pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); + } + if( appendBias ){ + pik_flags |= OPFLAG_APPEND; + } + if( useSeekResult ){ + pik_flags |= OPFLAG_USESEEKRESULT; + } + sqlite3VdbeAddOp3(v, OP_Insert, baseCur, regRec, regRowid); + if( !pParse->nested ){ + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + } + sqlite3VdbeChangeP5(v, pik_flags); } /* -** Generate code that will tell the VDBE the names of columns -** in the result set. This information is used to provide the -** azCol[] values in the callback. +** Generate code that will open cursors for a table and for all +** indices of that table. The "baseCur" parameter is the cursor number used +** for the table. Indices are opened on subsequent cursors. +** +** Return the number of indices on the table. */ -static void generateColumnNames( - Parse *pParse, /* Parser context */ - SrcList *pTabList, /* List of tables */ - ExprList *pEList /* Expressions defining the result set */ +SQLITE_PRIVATE int sqlite3OpenTableAndIndices( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table to be opened */ + int baseCur, /* Cursor number assigned to the table */ + int op /* OP_OpenRead or OP_OpenWrite */ ){ - Vdbe *v = pParse->pVdbe; - int i, j; - sqlite3 *db = pParse->db; - int fullNames, shortNames; + int i; + int iDb; + Index *pIdx; + Vdbe *v; -#ifndef SQLITE_OMIT_EXPLAIN - /* If this is an EXPLAIN, skip this step */ - if( pParse->explain ){ - return; + if( IsVirtual(pTab) ) return 0; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + sqlite3OpenTable(pParse, baseCur, iDb, pTab, op); + for(i=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + assert( pIdx->pSchema==pTab->pSchema ); + sqlite3VdbeAddOp4(v, op, i+baseCur, pIdx->tnum, iDb, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); } -#endif - - if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; - pParse->colNamesSet = 1; - fullNames = (db->flags & SQLITE_FullColNames)!=0; - shortNames = (db->flags & SQLITE_ShortColNames)!=0; - sqlite3VdbeSetNumCols(v, pEList->nExpr); - for(i=0; inExpr; i++){ - Expr *p; - p = pEList->a[i].pExpr; - if( NEVER(p==0) ) continue; - if( pEList->a[i].zName ){ - char *zName = pEList->a[i].zName; - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ - Table *pTab; - char *zCol; - int iCol = p->iColumn; - for(j=0; ALWAYS(jnSrc); j++){ - if( pTabList->a[j].iCursor==p->iTable ) break; - } - assert( jnSrc ); - pTab = pTabList->a[j].pTab; - if( iCol<0 ) iCol = pTab->iPKey; - assert( iCol==-1 || (iCol>=0 && iColnCol) ); - if( iCol<0 ){ - zCol = "rowid"; - }else{ - zCol = pTab->aCol[iCol].zName; - } - if( !shortNames && !fullNames ){ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, - sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); - }else if( fullNames ){ - char *zName = 0; - zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); - }else{ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); - } - }else{ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, - sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); - } + if( pParse->nTabnTab = baseCur+i; } - generateColumnTypes(pParse, pTabList, pEList); + return i-1; } + +#ifdef SQLITE_TEST /* -** Given a an expression list (which is really the list of expressions -** that form the result set of a SELECT statement) compute appropriate -** column names for a table that would hold the expression list. -** -** All column names will be unique. -** -** Only the column names are computed. Column.zType, Column.zColl, -** and other fields of Column are zeroed. -** -** Return SQLITE_OK on success. If a memory allocation error occurs, -** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. -*/ -static int selectColumnsFromExprList( - Parse *pParse, /* Parsing context */ - ExprList *pEList, /* Expr list from which to derive column names */ - int *pnCol, /* Write the number of columns here */ - Column **paCol /* Write the new column list here */ -){ - sqlite3 *db = pParse->db; /* Database connection */ - int i, j; /* Loop counters */ - int cnt; /* Index added to make the name unique */ - Column *aCol, *pCol; /* For looping over result columns */ - int nCol; /* Number of columns in the result set */ - Expr *p; /* Expression for a single result column */ - char *zName; /* Column name */ - int nName; /* Size of name in zName[] */ - - *pnCol = nCol = pEList->nExpr; - aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); - if( aCol==0 ) return SQLITE_NOMEM; - for(i=0, pCol=aCol; ia[i].pExpr; - assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) - || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); - if( (zName = pEList->a[i].zName)!=0 ){ - /* If the column contains an "AS " phrase, use as the name */ - zName = sqlite3DbStrDup(db, zName); - }else{ - Expr *pColExpr = p; /* The expression that is the result column name */ - Table *pTab; /* Table associated with this expression */ - while( pColExpr->op==TK_DOT ) pColExpr = pColExpr->pRight; - if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ - /* For columns use the column name name */ - int iCol = pColExpr->iColumn; - pTab = pColExpr->pTab; - if( iCol<0 ) iCol = pTab->iPKey; - zName = sqlite3MPrintf(db, "%s", - iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); - }else if( pColExpr->op==TK_ID ){ - assert( !ExprHasProperty(pColExpr, EP_IntValue) ); - zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken); - }else{ - /* Use the original text of the column expression as its name */ - zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan); - } - } - if( db->mallocFailed ){ - sqlite3DbFree(db, zName); - break; - } - - /* Make sure the column name is unique. If the name is not unique, - ** append a integer to the name so that it becomes unique. - */ - nName = sqlite3Strlen30(zName); - for(j=cnt=0; jzName = zName; - } - if( db->mallocFailed ){ - for(j=0; jdb; - NameContext sNC; - Column *pCol; - CollSeq *pColl; - int i; - Expr *p; - struct ExprList_item *a; +SQLITE_API int sqlite3_xferopt_count; +#endif /* SQLITE_TEST */ - assert( pSelect!=0 ); - assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( nCol==pSelect->pEList->nExpr || db->mallocFailed ); - if( db->mallocFailed ) return; - memset(&sNC, 0, sizeof(sNC)); - sNC.pSrcList = pSelect->pSrc; - a = pSelect->pEList->a; - for(i=0, pCol=aCol; izType = sqlite3DbStrDup(db, columnType(&sNC, p, 0, 0, 0)); - pCol->affinity = sqlite3ExprAffinity(p); - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; - pColl = sqlite3ExprCollSeq(pParse, p); - if( pColl ){ - pCol->zColl = sqlite3DbStrDup(db, pColl->zName); - } - } -} +#ifndef SQLITE_OMIT_XFER_OPT /* -** Given a SELECT statement, generate a Table structure that describes -** the result set of that SELECT. +** Check to collation names to see if they are compatible. */ -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ - Table *pTab; - sqlite3 *db = pParse->db; - int savedFlags; - - savedFlags = db->flags; - db->flags &= ~SQLITE_FullColNames; - db->flags |= SQLITE_ShortColNames; - sqlite3SelectPrep(pParse, pSelect, 0); - if( pParse->nErr ) return 0; - while( pSelect->pPrior ) pSelect = pSelect->pPrior; - db->flags = savedFlags; - pTab = sqlite3DbMallocZero(db, sizeof(Table) ); - if( pTab==0 ){ - return 0; +static int xferCompatibleCollation(const char *z1, const char *z2){ + if( z1==0 ){ + return z2==0; } - /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled */ - assert( db->lookaside.bEnabled==0 ); - pTab->nRef = 1; - pTab->zName = 0; - pTab->nRowEst = 1000000; - selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); - pTab->iPKey = -1; - if( db->mallocFailed ){ - sqlite3DeleteTable(db, pTab); + if( z2==0 ){ return 0; } - return pTab; -} - -/* -** Get a VDBE for the given parser context. Create a new one if necessary. -** If an error occurs, return NULL and leave a message in pParse. -*/ -SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe; - if( v==0 ){ - v = pParse->pVdbe = sqlite3VdbeCreate(pParse->db); -#ifndef SQLITE_OMIT_TRACE - if( v ){ - sqlite3VdbeAddOp0(v, OP_Trace); - } -#endif - } - return v; + return sqlite3StrICmp(z1, z2)==0; } /* -** Compute the iLimit and iOffset fields of the SELECT based on the -** pLimit and pOffset expressions. pLimit and pOffset hold the expressions -** that appear in the original SQL statement after the LIMIT and OFFSET -** keywords. Or NULL if those keywords are omitted. iLimit and iOffset -** are the integer memory register numbers for counters used to compute -** the limit and offset. If there is no limit and/or offset, then -** iLimit and iOffset are negative. +** Check to see if index pSrc is compatible as a source of data +** for index pDest in an insert transfer optimization. The rules +** for a compatible index: ** -** This routine changes the values of iLimit and iOffset only if -** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values -** (usually but not always -1) prior to calling this routine. -** Only if pLimit!=0 or pOffset!=0 do the limit registers get -** redefined. The UNION ALL operator uses this property to force -** the reuse of the same limit and offset registers across multiple -** SELECT statements. +** * The index is over the same set of columns +** * The same DESC and ASC markings occurs on all columns +** * The same onError processing (OE_Abort, OE_Ignore, etc) +** * The same collating sequence on each column */ -static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ - Vdbe *v = 0; - int iLimit = 0; - int iOffset; - int addr1, n; - if( p->iLimit ) return; - - /* - ** "LIMIT -1" always shows all rows. There is some - ** contraversy about what the correct behavior should be. - ** The current implementation interprets "LIMIT 0" to mean - ** no rows. - */ - sqlite3ExprCacheClear(pParse); - assert( p->pOffset==0 || p->pLimit!=0 ); - if( p->pLimit ){ - p->iLimit = iLimit = ++pParse->nMem; - v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ - if( sqlite3ExprIsInteger(p->pLimit, &n) ){ - sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); - VdbeComment((v, "LIMIT counter")); - if( n==0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); - }else{ - if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n; - } - }else{ - sqlite3ExprCode(pParse, p->pLimit, iLimit); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); - VdbeComment((v, "LIMIT counter")); - sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); +static int xferCompatibleIndex(Index *pDest, Index *pSrc){ + int i; + assert( pDest && pSrc ); + assert( pDest->pTable!=pSrc->pTable ); + if( pDest->nColumn!=pSrc->nColumn ){ + return 0; /* Different number of columns */ + } + if( pDest->onError!=pSrc->onError ){ + return 0; /* Different conflict resolution strategies */ + } + for(i=0; inColumn; i++){ + if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ + return 0; /* Different columns indexed */ } - if( p->pOffset ){ - p->iOffset = iOffset = ++pParse->nMem; - pParse->nMem++; /* Allocate an extra register for limit+offset */ - sqlite3ExprCode(pParse, p->pOffset, iOffset); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); - VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); - VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); - sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); - sqlite3VdbeJumpHere(v, addr1); + if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ + return 0; /* Different sort orders */ + } + if( !xferCompatibleCollation(pSrc->azColl[i],pDest->azColl[i]) ){ + return 0; /* Different collating sequences */ } } -} -#ifndef SQLITE_OMIT_COMPOUND_SELECT -/* -** Return the appropriate collating sequence for the iCol-th column of -** the result set for the compound-select statement "p". Return NULL if -** the column has no default collating sequence. -** -** The collating sequence for the compound select is taken from the -** left-most term of the select that has a collating sequence. -*/ -static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ - CollSeq *pRet; - if( p->pPrior ){ - pRet = multiSelectCollSeq(pParse, p->pPrior, iCol); - }else{ - pRet = 0; - } - assert( iCol>=0 ); - if( pRet==0 && iColpEList->nExpr ){ - pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); - } - return pRet; + /* If no test above fails then the indices must be compatible */ + return 1; } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - -/* Forward reference */ -static int multiSelectOrderBy( - Parse *pParse, /* Parsing context */ - Select *p, /* The right-most of SELECTs to be coded */ - SelectDest *pDest /* What to do with query results */ -); - -#ifndef SQLITE_OMIT_COMPOUND_SELECT /* -** This routine is called to process a compound query form from -** two or more separate queries using UNION, UNION ALL, EXCEPT, or -** INTERSECT +** Attempt the transfer optimization on INSERTs of the form ** -** "p" points to the right-most of the two queries. the query on the -** left is p->pPrior. The left query could also be a compound query -** in which case this routine will be called recursively. +** INSERT INTO tab1 SELECT * FROM tab2; ** -** The results of the total query are to be written into a destination -** of type eDest with parameter iParm. +** This optimization is only attempted if ** -** Example 1: Consider a three-way compound SQL statement. +** (1) tab1 and tab2 have identical schemas including all the +** same indices and constraints ** -** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3 +** (2) tab1 and tab2 are different tables ** -** This statement is parsed up as follows: +** (3) There must be no triggers on tab1 ** -** SELECT c FROM t3 -** | -** `-----> SELECT b FROM t2 -** | -** `------> SELECT a FROM t1 +** (4) The result set of the SELECT statement is "*" ** -** The arrows in the diagram above represent the Select.pPrior pointer. -** So if this routine is called with p equal to the t3 query, then -** pPrior will be the t2 query. p->op will be TK_UNION in this case. +** (5) The SELECT statement has no WHERE, HAVING, ORDER BY, GROUP BY, +** or LIMIT clause. ** -** Notice that because of the way SQLite parses compound SELECTs, the -** individual selects always group from left to right. +** (6) The SELECT statement is a simple (not a compound) select that +** contains only tab2 in its FROM clause +** +** This method for implementing the INSERT transfers raw records from +** tab2 over to tab1. The columns are not decoded. Raw records from +** the indices of tab2 are transfered to tab1 as well. In so doing, +** the resulting tab1 has much less fragmentation. +** +** This routine returns TRUE if the optimization is attempted. If any +** of the conditions above fail so that the optimization should not +** be attempted, then this routine returns FALSE. */ -static int multiSelect( - Parse *pParse, /* Parsing context */ - Select *p, /* The right-most of SELECTs to be coded */ - SelectDest *pDest /* What to do with query results */ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ ){ - int rc = SQLITE_OK; /* Success code from a subroutine */ - Select *pPrior; /* Another SELECT immediately to our left */ - Vdbe *v; /* Generate code to this VDBE */ - SelectDest dest; /* Alternative data destination */ - Select *pDelete = 0; /* Chain of simple selects to delete */ - sqlite3 *db; /* Database connection */ -#ifndef SQLITE_OMIT_EXPLAIN - int iSub1; /* EQP id of left-hand query */ - int iSub2; /* EQP id of right-hand query */ -#endif + ExprList *pEList; /* The result set of the SELECT */ + Table *pSrc; /* The table in the FROM clause of SELECT */ + Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ + struct SrcList_item *pItem; /* An element of pSelect->pSrc */ + int i; /* Loop counter */ + int iDbSrc; /* The database of pSrc */ + int iSrc, iDest; /* Cursors from source and destination */ + int addr1, addr2; /* Loop addresses */ + int emptyDestTest; /* Address of test for empty pDest */ + int emptySrcTest; /* Address of test for empty pSrc */ + Vdbe *v; /* The VDBE we are building */ + KeyInfo *pKey; /* Key information for an index */ + int regAutoinc; /* Memory register used by AUTOINC */ + int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ + int regData, regRowid; /* Registers holding data and rowid */ - /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only - ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. - */ - assert( p && p->pPrior ); /* Calling function guarantees this much */ - db = pParse->db; - pPrior = p->pPrior; - assert( pPrior->pRightmost!=pPrior ); - assert( pPrior->pRightmost==p->pRightmost ); - dest = *pDest; - if( pPrior->pOrderBy ){ - sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", - selectOpName(p->op)); - rc = 1; - goto multi_select_end; + if( pSelect==0 ){ + return 0; /* Must be of the form INSERT INTO ... SELECT ... */ } - if( pPrior->pLimit ){ - sqlite3ErrorMsg(pParse,"LIMIT clause should come after %s not before", - selectOpName(p->op)); - rc = 1; - goto multi_select_end; + if( sqlite3TriggerList(pParse, pDest) ){ + return 0; /* tab1 must not have triggers */ } - - v = sqlite3GetVdbe(pParse); - assert( v!=0 ); /* The VDBE already created by calling function */ - - /* Create the destination temporary table if necessary - */ - if( dest.eDest==SRT_EphemTab ){ - assert( p->pEList ); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - dest.eDest = SRT_Table; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pDest->tabFlags & TF_Virtual ){ + return 0; /* tab1 must not be a virtual table */ } - - /* Make sure all SELECTs in the statement have the same number of elements - ** in their result sets. - */ - assert( p->pEList && pPrior->pEList ); - if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); - rc = 1; - goto multi_select_end; +#endif + if( onError==OE_Default ){ + onError = OE_Abort; } - - /* Compound SELECTs that have an ORDER BY clause are handled separately. - */ - if( p->pOrderBy ){ - return multiSelectOrderBy(pParse, p, pDest); + if( onError!=OE_Abort && onError!=OE_Rollback ){ + return 0; /* Cannot do OR REPLACE or OR IGNORE or OR FAIL */ + } + assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ + if( pSelect->pSrc->nSrc!=1 ){ + return 0; /* FROM clause must have exactly one term */ + } + if( pSelect->pSrc->a[0].pSelect ){ + return 0; /* FROM clause cannot contain a subquery */ + } + if( pSelect->pWhere ){ + return 0; /* SELECT may not have a WHERE clause */ + } + if( pSelect->pOrderBy ){ + return 0; /* SELECT may not have an ORDER BY clause */ + } + /* Do not need to test for a HAVING clause. If HAVING is present but + ** there is no ORDER BY, we will get an error. */ + if( pSelect->pGroupBy ){ + return 0; /* SELECT may not have a GROUP BY clause */ + } + if( pSelect->pLimit ){ + return 0; /* SELECT may not have a LIMIT clause */ + } + assert( pSelect->pOffset==0 ); /* Must be so if pLimit==0 */ + if( pSelect->pPrior ){ + return 0; /* SELECT may not be a compound query */ + } + if( pSelect->selFlags & SF_Distinct ){ + return 0; /* SELECT may not be DISTINCT */ + } + pEList = pSelect->pEList; + assert( pEList!=0 ); + if( pEList->nExpr!=1 ){ + return 0; /* The result set must have exactly one column */ + } + assert( pEList->a[0].pExpr ); + if( pEList->a[0].pExpr->op!=TK_ALL ){ + return 0; /* The result set must be the special operator "*" */ } - /* Generate code for the left and right SELECT statements. + /* At this point we have established that the statement is of the + ** correct syntactic form to participate in this optimization. Now + ** we have to check the semantics. */ - switch( p->op ){ - case TK_ALL: { - int addr = 0; - int nLimit; - assert( !pPrior->pLimit ); - pPrior->pLimit = p->pLimit; - pPrior->pOffset = p->pOffset; - explainSetInteger(iSub1, pParse->iNextSelectId); - rc = sqlite3Select(pParse, pPrior, &dest); - p->pLimit = 0; - p->pOffset = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); - VdbeComment((v, "Jump ahead if LIMIT reached")); - } - explainSetInteger(iSub2, pParse->iNextSelectId); - rc = sqlite3Select(pParse, p, &dest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->nSelectRow += pPrior->nSelectRow; - if( pPrior->pLimit - && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && p->nSelectRow > (double)nLimit - ){ - p->nSelectRow = (double)nLimit; - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); - } - break; + pItem = pSelect->pSrc->a; + pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + if( pSrc==0 ){ + return 0; /* FROM clause does not contain a real table */ + } + if( pSrc==pDest ){ + return 0; /* tab1 and tab2 may not be the same table */ + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pSrc->tabFlags & TF_Virtual ){ + return 0; /* tab2 must not be a virtual table */ + } +#endif + if( pSrc->pSelect ){ + return 0; /* tab2 may not be a view */ + } + if( pDest->nCol!=pSrc->nCol ){ + return 0; /* Number of columns must be the same in tab1 and tab2 */ + } + if( pDest->iPKey!=pSrc->iPKey ){ + return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ + } + for(i=0; inCol; i++){ + if( pDest->aCol[i].affinity!=pSrc->aCol[i].affinity ){ + return 0; /* Affinity must be the same on all columns */ } - case TK_EXCEPT: - case TK_UNION: { - int unionTab; /* Cursor number of the temporary table holding result */ - u8 op = 0; /* One of the SRT_ operations to apply to self */ - int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ - int addr; - SelectDest uniondest; + if( !xferCompatibleCollation(pDest->aCol[i].zColl, pSrc->aCol[i].zColl) ){ + return 0; /* Collating sequence must be the same on all columns */ + } + if( pDest->aCol[i].notNull && !pSrc->aCol[i].notNull ){ + return 0; /* tab2 must be NOT NULL if tab1 is */ + } + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + if( pDestIdx->onError!=OE_None ){ + destHasUniqueIdx = 1; + } + for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + if( pSrcIdx==0 ){ + return 0; /* pDestIdx has no corresponding index in pSrc */ + } + } +#ifndef SQLITE_OMIT_CHECK + if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + return 0; /* Tables have different CHECK constraints. Ticket #2252 */ + } +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + /* Disallow the transfer optimization if the destination table constains + ** any foreign key constraints. This is more restrictive than necessary. + ** But the main beneficiary of the transfer optimization is the VACUUM + ** command, and the VACUUM command disables foreign key constraints. So + ** the extra complication to make this rule less restrictive is probably + ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] + */ + if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + return 0; + } +#endif - testcase( p->op==TK_EXCEPT ); - testcase( p->op==TK_UNION ); - priorOp = SRT_Union; - if( dest.eDest==priorOp && ALWAYS(!p->pLimit &&!p->pOffset) ){ - /* We can reuse a temporary table generated by a SELECT to our - ** right. - */ - assert( p->pRightmost!=p ); /* Can only happen for leftward elements - ** of a 3-way or more compound */ - assert( p->pLimit==0 ); /* Not allowed on leftward elements */ - assert( p->pOffset==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iParm; - }else{ - /* We will need to create our own temporary table to hold the - ** intermediate results. - */ - unionTab = pParse->nTab++; - assert( p->pOrderBy==0 ); - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - p->pRightmost->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - } + /* If we get this far, it means either: + ** + ** * We can always do the transfer if the table contains an + ** an integer primary key + ** + ** * We can conditionally do the transfer if the destination + ** table is empty. + */ +#ifdef SQLITE_TEST + sqlite3_xferopt_count++; +#endif + iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); + v = sqlite3GetVdbe(pParse); + sqlite3CodeVerifySchema(pParse, iDbSrc); + iSrc = pParse->nTab++; + iDest = pParse->nTab++; + regAutoinc = autoIncBegin(pParse, iDbDest, pDest); + sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); + if( (pDest->iPKey<0 && pDest->pIndex!=0) || destHasUniqueIdx ){ + /* If tables do not have an INTEGER PRIMARY KEY and there + ** are indices to be copied and the destination is not empty, + ** we have to disallow the transfer optimization because the + ** the rowids might change which will mess up indexing. + ** + ** Or if the destination has a UNIQUE index and is not empty, + ** we also disallow the transfer optimization because we cannot + ** insure that all entries in the union of DEST and SRC will be + ** unique. + */ + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); + emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + sqlite3VdbeJumpHere(v, addr1); + }else{ + emptyDestTest = 0; + } + sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + regData = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + if( pDest->iPKey>=0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + sqlite3HaltConstraint( + pParse, onError, "PRIMARY KEY must be unique", P4_STATIC); + sqlite3VdbeJumpHere(v, addr2); + autoIncStep(pParse, regAutoinc, regRowid); + }else if( pDest->pIndex==0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); + }else{ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + } + sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + assert( pSrcIdx ); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + pKey = sqlite3IndexKeyinfo(pParse, pSrcIdx); + sqlite3VdbeAddOp4(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pSrcIdx->zName)); + pKey = sqlite3IndexKeyinfo(pParse, pDestIdx); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pDestIdx->zName)); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); + sqlite3VdbeJumpHere(v, addr1); + } + sqlite3VdbeJumpHere(v, emptySrcTest); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regData); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + if( emptyDestTest ){ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); + sqlite3VdbeJumpHere(v, emptyDestTest); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + return 0; + }else{ + return 1; + } +} +#endif /* SQLITE_OMIT_XFER_OPT */ - /* Code the SELECT statements to our left - */ - assert( !pPrior->pOrderBy ); - sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - explainSetInteger(iSub1, pParse->iNextSelectId); - rc = sqlite3Select(pParse, pPrior, &uniondest); - if( rc ){ - goto multi_select_end; - } +/************** End of insert.c **********************************************/ +/************** Begin file legacy.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ - /* Code the current SELECT statement - */ - if( p->op==TK_EXCEPT ){ - op = SRT_Except; - }else{ - assert( p->op==TK_UNION ); - op = SRT_Union; - } - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; - uniondest.eDest = op; - explainSetInteger(iSub2, pParse->iNextSelectId); - rc = sqlite3Select(pParse, p, &uniondest); - testcase( rc!=SQLITE_OK ); - /* Query flattening in sqlite3Select() might refill p->pOrderBy. - ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ - sqlite3ExprListDelete(db, p->pOrderBy); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->pOrderBy = 0; - if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->pOffset = pOffset; - p->iLimit = 0; - p->iOffset = 0; - /* Convert the data in the temporary table into whatever form - ** it is that we currently need. - */ - assert( unionTab==dest.iParm || dest.eDest!=priorOp ); - if( dest.eDest!=priorOp ){ - int iCont, iBreak, iStart; - assert( p->pEList ); - if( dest.eDest==SRT_Output ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); - } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); - iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); - } - break; +/* +** Execute SQL code. Return one of the SQLITE_ success/failure +** codes. Also write an error message into memory obtained from +** malloc() and make *pzErrMsg point to that message. +** +** If the SQL is a query, then for each row in the query result +** the xCallback() function is called. pArg becomes the first +** argument to xCallback(). If xCallback=NULL then no callback +** is invoked, even for queries. +*/ +SQLITE_API int sqlite3_exec( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + sqlite3_callback xCallback, /* Invoke this callback routine */ + void *pArg, /* First argument to xCallback() */ + char **pzErrMsg /* Write error messages here */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3_stmt *pStmt = 0; /* The current SQL statement */ + char **azCols = 0; /* Names of result columns */ + int nRetry = 0; /* Number of retry attempts */ + int callbackIsInit; /* True if callback data is initialized */ + + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; + if( zSql==0 ) zSql = ""; + + sqlite3_mutex_enter(db->mutex); + sqlite3Error(db, SQLITE_OK, 0); + while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + int nCol; + char **azVals = 0; + + pStmt = 0; + rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc!=SQLITE_OK ){ + continue; + } + if( !pStmt ){ + /* this happens for a comment or white-space */ + zSql = zLeftover; + continue; } - default: assert( p->op==TK_INTERSECT ); { - int tab1, tab2; - int iCont, iBreak, iStart; - Expr *pLimit, *pOffset; - int addr; - SelectDest intersectdest; - int r1; - /* INTERSECT is different from the others since it requires - ** two temporary tables. Hence it has its own case. Begin - ** by allocating the tables we will need. - */ - tab1 = pParse->nTab++; - tab2 = pParse->nTab++; - assert( p->pOrderBy==0 ); + callbackIsInit = 0; + nCol = sqlite3_column_count(pStmt); - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - p->pRightmost->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); + while( 1 ){ + int i; + rc = sqlite3_step(pStmt); - /* Code the SELECTs to our left into temporary table "tab1". - */ - sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - explainSetInteger(iSub1, pParse->iNextSelectId); - rc = sqlite3Select(pParse, pPrior, &intersectdest); - if( rc ){ - goto multi_select_end; + /* Invoke the callback function if required */ + if( xCallback && (SQLITE_ROW==rc || + (SQLITE_DONE==rc && !callbackIsInit + && db->flags&SQLITE_NullCallback)) ){ + if( !callbackIsInit ){ + azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + if( azCols==0 ){ + goto exec_out; + } + for(i=0; imallocFailed = 1; + goto exec_out; + } + } + } + if( xCallback(pArg, nCol, azVals, azCols) ){ + rc = SQLITE_ABORT; + sqlite3VdbeFinalize((Vdbe *)pStmt); + pStmt = 0; + sqlite3Error(db, SQLITE_ABORT, 0); + goto exec_out; + } } - /* Code the current SELECT into temporary table "tab2" - */ - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); - assert( p->addrOpenEphm[1] == -1 ); - p->addrOpenEphm[1] = addr; - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; - intersectdest.iParm = tab2; - explainSetInteger(iSub2, pParse->iNextSelectId); - rc = sqlite3Select(pParse, p, &intersectdest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->pOffset = pOffset; - - /* Generate code to take the intersection of the two temporary - ** tables. - */ - assert( p->pEList ); - if( dest.eDest==SRT_Output ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); + if( rc!=SQLITE_ROW ){ + rc = sqlite3VdbeFinalize((Vdbe *)pStmt); + pStmt = 0; + if( rc!=SQLITE_SCHEMA ){ + nRetry = 0; + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; + } + break; } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); - r1 = sqlite3GetTempReg(pParse); - iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); - sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - 0, -1, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); - sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); - break; } - } - explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL); + sqlite3DbFree(db, azCols); + azCols = 0; + } - /* Compute collating sequences used by - ** temporary tables needed to implement the compound select. - ** Attach the KeyInfo structure to all temporary tables. - ** - ** This section is run by the right-most SELECT statement only. - ** SELECT statements to the left always skip this part. The right-most - ** SELECT might also skip this part if it has no ORDER BY clause and - ** no temp tables are required. - */ - if( p->selFlags & SF_UsesEphemeral ){ - int i; /* Loop counter */ - KeyInfo *pKeyInfo; /* Collating sequence for the result set */ - Select *pLoop; /* For looping through SELECT statements */ - CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ - int nCol; /* Number of columns in result set */ +exec_out: + if( pStmt ) sqlite3VdbeFinalize((Vdbe *)pStmt); + sqlite3DbFree(db, azCols); - assert( p->pRightmost==p ); - nCol = p->pEList->nExpr; - pKeyInfo = sqlite3DbMallocZero(db, - sizeof(*pKeyInfo)+nCol*(sizeof(CollSeq*) + 1)); - if( !pKeyInfo ){ + rc = sqlite3ApiExit(db, rc); + if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ + int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); + *pzErrMsg = sqlite3Malloc(nErrMsg); + if( *pzErrMsg ){ + memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); + }else{ rc = SQLITE_NOMEM; - goto multi_select_end; - } - - pKeyInfo->enc = ENC(db); - pKeyInfo->nField = (u16)nCol; - - for(i=0, apColl=pKeyInfo->aColl; ipDfltColl; - } - } - - for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ - for(i=0; i<2; i++){ - int addr = pLoop->addrOpenEphm[i]; - if( addr<0 ){ - /* If [0] is unused then [1] is also unused. So we can - ** always safely abort as soon as the first unused slot is found */ - assert( pLoop->addrOpenEphm[1]<0 ); - break; - } - sqlite3VdbeChangeP2(v, addr, nCol); - sqlite3VdbeChangeP4(v, addr, (char*)pKeyInfo, P4_KEYINFO); - pLoop->addrOpenEphm[i] = -1; - } + sqlite3Error(db, SQLITE_NOMEM, 0); } - sqlite3DbFree(db, pKeyInfo); + }else if( pzErrMsg ){ + *pzErrMsg = 0; } -multi_select_end: - pDest->iMem = dest.iMem; - pDest->nMem = dest.nMem; - sqlite3SelectDelete(db, pDelete); + assert( (rc&db->errMask)==rc ); + sqlite3_mutex_leave(db->mutex); return rc; } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ +/************** End of legacy.c **********************************************/ +/************** Begin file loadext.c *****************************************/ /* -** Code an output subroutine for a coroutine implementation of a -** SELECT statment. +** 2006 June 7 ** -** The data to be output is contained in pIn->iMem. There are -** pIn->nMem columns to be output. pDest is where the output should -** be sent. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** regReturn is the number of the register holding the subroutine -** return address. -** -** If regPrev>0 then it is the first register in a vector that -** records the previous output. mem[regPrev] is a flag that is false -** if there has been no previous output. If regPrev>0 then code is -** generated to suppress duplicates. pKeyInfo is used for comparing -** keys. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** If the LIMIT found in p->iLimit is reached, jump immediately to -** iBreak. +************************************************************************* +** This file contains code used to dynamically load extensions into +** the SQLite library. */ -static int generateOutputSubroutine( - Parse *pParse, /* Parsing context */ - Select *p, /* The SELECT statement */ - SelectDest *pIn, /* Coroutine supplying data */ - SelectDest *pDest, /* Where to send the data */ - int regReturn, /* The return address register */ - int regPrev, /* Previous result register. No uniqueness if 0 */ - KeyInfo *pKeyInfo, /* For comparing with previous entry */ - int p4type, /* The p4 type for pKeyInfo */ - int iBreak /* Jump here if we hit the LIMIT */ -){ - Vdbe *v = pParse->pVdbe; - int iContinue; - int addr; - - addr = sqlite3VdbeCurrentAddr(v); - iContinue = sqlite3VdbeMakeLabel(v); - - /* Suppress duplicates for UNION, EXCEPT, and INTERSECT - */ - if( regPrev ){ - int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iMem, regPrev+1, pIn->nMem, - (char*)pKeyInfo, p4type); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); - sqlite3VdbeJumpHere(v, j1); - sqlite3ExprCodeCopy(pParse, pIn->iMem, regPrev+1, pIn->nMem); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); - } - if( pParse->db->mallocFailed ) return 0; - - /* Suppress the the first OFFSET entries if there is an OFFSET clause - */ - codeOffset(v, p, iContinue); - - switch( pDest->eDest ){ - /* Store the result as data using a unique key. - */ - case SRT_Table: - case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); - int r2 = sqlite3GetTempReg(pParse); - testcase( pDest->eDest==SRT_Table ); - testcase( pDest->eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iMem, pIn->nMem, r1); - sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, pDest->iParm, r1, r2); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3ReleaseTempReg(pParse, r2); - sqlite3ReleaseTempReg(pParse, r1); - break; - } - -#ifndef SQLITE_OMIT_SUBQUERY - /* If we are creating a set for an "expr IN (SELECT ...)" construct, - ** then there should be a single item on the stack. Write this - ** item into the set table with bogus data. - */ - case SRT_Set: { - int r1; - assert( pIn->nMem==1 ); - p->affinity = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affinity); - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iMem, 1, r1, &p->affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iParm, r1); - sqlite3ReleaseTempReg(pParse, r1); - break; - } -#if 0 /* Never occurs on an ORDER BY query */ - /* If any row exist in the result set, record that fact and abort. - */ - case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iParm); - /* The LIMIT clause will terminate the loop for us */ - break; - } +#ifndef SQLITE_CORE + #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ #endif - - /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell and break out - ** of the scan loop. - */ - case SRT_Mem: { - assert( pIn->nMem==1 ); - sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iParm, 1); - /* The LIMIT clause will jump out of the loop for us */ - break; - } -#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ - - /* The results are stored in a sequence of registers - ** starting at pDest->iMem. Then the co-routine yields. - */ - case SRT_Coroutine: { - if( pDest->iMem==0 ){ - pDest->iMem = sqlite3GetTempRange(pParse, pIn->nMem); - pDest->nMem = pIn->nMem; - } - sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iMem, pDest->nMem); - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); - break; - } - - /* If none of the above, then the result destination must be - ** SRT_Output. This routine is never called with any other - ** destination other than the ones handled above or SRT_Output. - ** - ** For SRT_Output, results are stored in a sequence of registers. - ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to - ** return the next row of result. - */ - default: { - assert( pDest->eDest==SRT_Output ); - sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iMem, pIn->nMem); - sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, pIn->nMem); - break; - } - } - - /* Jump to the end of the loop if the LIMIT is reached. - */ - if( p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); - } - - /* Generate the subroutine return - */ - sqlite3VdbeResolveLabel(v, iContinue); - sqlite3VdbeAddOp1(v, OP_Return, regReturn); - - return addr; -} - +/************** Include sqlite3ext.h in the middle of loadext.c **************/ +/************** Begin file sqlite3ext.h **************************************/ /* -** Alternative compound select code generator for cases when there -** is an ORDER BY clause. -** -** We assume a query of the following form: -** -** ORDER BY -** -** is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea -** is to code both and with the ORDER BY clause as -** co-routines. Then run the co-routines in parallel and merge the results -** into the output. In addition to the two coroutines (called selectA and -** selectB) there are 7 subroutines: -** -** outA: Move the output of the selectA coroutine into the output -** of the compound query. -** -** outB: Move the output of the selectB coroutine into the output -** of the compound query. (Only generated for UNION and -** UNION ALL. EXCEPT and INSERTSECT never output a row that -** appears only in B.) -** -** AltB: Called when there is data from both coroutines and AB. -** -** EofA: Called when data is exhausted from selectA. -** -** EofB: Called when data is exhausted from selectB. -** -** The implementation of the latter five subroutines depend on which -** is used: -** -** -** UNION ALL UNION EXCEPT INTERSECT -** ------------- ----------------- -------------- ----------------- -** AltB: outA, nextA outA, nextA outA, nextA nextA -** -** AeqB: outA, nextA nextA nextA outA, nextA -** -** AgtB: outB, nextB outB, nextB nextB nextB -** -** EofA: outB, nextB outB, nextB halt halt -** -** EofB: outA, nextA outA, nextA outA, nextA halt -** -** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA -** causes an immediate jump to EofA and an EOF on B following nextB causes -** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or -** following nextX causes a jump to the end of the select processing. -** -** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled -** within the output subroutine. The regPrev register set holds the previously -** output value. A comparison is made against this value and the output -** is skipped if the next results would be the same as the previous. +** 2006 June 7 ** -** The implementation plan is to implement the two coroutines and seven -** subroutines first, then put the control logic at the bottom. Like this: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** goto Init -** coA: coroutine for left query (A) -** coB: coroutine for right query (B) -** outA: output one row of A -** outB: output one row of B (UNION and UNION ALL only) -** EofA: ... -** EofB: ... -** AltB: ... -** AeqB: ... -** AgtB: ... -** Init: initialize coroutine registers -** yield coA -** if eof(A) goto EofA -** yield coB -** if eof(B) goto EofB -** Cmpr: Compare A, B -** Jump AltB, AeqB, AgtB -** End: ... +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not -** actually called using Gosub and they do not Return. EofA and EofB loop -** until all data is exhausted then jump to the "end" labe. AltB, AeqB, -** and AgtB jump to either L2 or to one of EofA or EofB. +************************************************************************* +** This header file defines the SQLite interface for use by +** shared libraries that want to be imported as extensions into +** an SQLite instance. Shared libraries that intend to be loaded +** as extensions by SQLite should #include this file instead of +** sqlite3.h. */ -#ifndef SQLITE_OMIT_COMPOUND_SELECT -static int multiSelectOrderBy( - Parse *pParse, /* Parsing context */ - Select *p, /* The right-most of SELECTs to be coded */ - SelectDest *pDest /* What to do with query results */ -){ - int i, j; /* Loop counters */ - Select *pPrior; /* Another SELECT immediately to our left */ - Vdbe *v; /* Generate code to this VDBE */ - SelectDest destA; /* Destination for coroutine A */ - SelectDest destB; /* Destination for coroutine B */ - int regAddrA; /* Address register for select-A coroutine */ - int regEofA; /* Flag to indicate when select-A is complete */ - int regAddrB; /* Address register for select-B coroutine */ - int regEofB; /* Flag to indicate when select-B is complete */ - int addrSelectA; /* Address of the select-A coroutine */ - int addrSelectB; /* Address of the select-B coroutine */ - int regOutA; /* Address register for the output-A subroutine */ - int regOutB; /* Address register for the output-B subroutine */ - int addrOutA; /* Address of the output-A subroutine */ - int addrOutB = 0; /* Address of the output-B subroutine */ - int addrEofA; /* Address of the select-A-exhausted subroutine */ - int addrEofB; /* Address of the select-B-exhausted subroutine */ - int addrAltB; /* Address of the AB subroutine */ - int regLimitA; /* Limit register for select-A */ - int regLimitB; /* Limit register for select-A */ - int regPrev; /* A range of registers to hold previous output */ - int savedLimit; /* Saved value of p->iLimit */ - int savedOffset; /* Saved value of p->iOffset */ - int labelCmpr; /* Label for the start of the merge algorithm */ - int labelEnd; /* Label for the end of the overall SELECT stmt */ - int j1; /* Jump instructions that get retargetted */ - int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ - KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ - KeyInfo *pKeyMerge; /* Comparison information for merging rows */ - sqlite3 *db; /* Database connection */ - ExprList *pOrderBy; /* The ORDER BY clause */ - int nOrderBy; /* Number of terms in the ORDER BY clause */ - int *aPermute; /* Mapping from ORDER BY terms to result set columns */ -#ifndef SQLITE_OMIT_EXPLAIN - int iSub1; /* EQP id of left-hand query */ - int iSub2; /* EQP id of right-hand query */ -#endif - - assert( p->pOrderBy!=0 ); - assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ - db = pParse->db; - v = pParse->pVdbe; - assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ - labelEnd = sqlite3VdbeMakeLabel(v); - labelCmpr = sqlite3VdbeMakeLabel(v); +#ifndef _SQLITE3EXT_H_ +#define _SQLITE3EXT_H_ +typedef struct sqlite3_api_routines sqlite3_api_routines; - /* Patch up the ORDER BY clause - */ - op = p->op; - pPrior = p->pPrior; - assert( pPrior->pOrderBy==0 ); - pOrderBy = p->pOrderBy; - assert( pOrderBy ); - nOrderBy = pOrderBy->nExpr; +/* +** The following structure holds pointers to all of the SQLite API +** routines. +** +** WARNING: In order to maintain backwards compatibility, add new +** interfaces to the end of this structure only. If you insert new +** interfaces in the middle of this structure, then older different +** versions of SQLite will not be able to load each others' shared +** libraries! +*/ +struct sqlite3_api_routines { + void * (*aggregate_context)(sqlite3_context*,int nBytes); + int (*aggregate_count)(sqlite3_context*); + int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); + int (*bind_double)(sqlite3_stmt*,int,double); + int (*bind_int)(sqlite3_stmt*,int,int); + int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); + int (*bind_null)(sqlite3_stmt*,int); + int (*bind_parameter_count)(sqlite3_stmt*); + int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); + const char * (*bind_parameter_name)(sqlite3_stmt*,int); + int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); + int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); + int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); + int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); + int (*busy_timeout)(sqlite3*,int ms); + int (*changes)(sqlite3*); + int (*close)(sqlite3*); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const void*)); + const void * (*column_blob)(sqlite3_stmt*,int iCol); + int (*column_bytes)(sqlite3_stmt*,int iCol); + int (*column_bytes16)(sqlite3_stmt*,int iCol); + int (*column_count)(sqlite3_stmt*pStmt); + const char * (*column_database_name)(sqlite3_stmt*,int); + const void * (*column_database_name16)(sqlite3_stmt*,int); + const char * (*column_decltype)(sqlite3_stmt*,int i); + const void * (*column_decltype16)(sqlite3_stmt*,int); + double (*column_double)(sqlite3_stmt*,int iCol); + int (*column_int)(sqlite3_stmt*,int iCol); + sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); + const char * (*column_name)(sqlite3_stmt*,int); + const void * (*column_name16)(sqlite3_stmt*,int); + const char * (*column_origin_name)(sqlite3_stmt*,int); + const void * (*column_origin_name16)(sqlite3_stmt*,int); + const char * (*column_table_name)(sqlite3_stmt*,int); + const void * (*column_table_name16)(sqlite3_stmt*,int); + const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); + const void * (*column_text16)(sqlite3_stmt*,int iCol); + int (*column_type)(sqlite3_stmt*,int iCol); + sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); + void * (*commit_hook)(sqlite3*,int(*)(void*),void*); + int (*complete)(const char*sql); + int (*complete16)(const void*sql); + int (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); + int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); + int (*data_count)(sqlite3_stmt*pStmt); + sqlite3 * (*db_handle)(sqlite3_stmt*); + int (*declare_vtab)(sqlite3*,const char*); + int (*enable_shared_cache)(int); + int (*errcode)(sqlite3*db); + const char * (*errmsg)(sqlite3*); + const void * (*errmsg16)(sqlite3*); + int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); + int (*expired)(sqlite3_stmt*); + int (*finalize)(sqlite3_stmt*pStmt); + void (*free)(void*); + void (*free_table)(char**result); + int (*get_autocommit)(sqlite3*); + void * (*get_auxdata)(sqlite3_context*,int); + int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); + int (*global_recover)(void); + void (*interruptx)(sqlite3*); + sqlite_int64 (*last_insert_rowid)(sqlite3*); + const char * (*libversion)(void); + int (*libversion_number)(void); + void *(*malloc)(int); + char * (*mprintf)(const char*,...); + int (*open)(const char*,sqlite3**); + int (*open16)(const void*,sqlite3**); + int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); + void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); + void *(*realloc)(void*,int); + int (*reset)(sqlite3_stmt*pStmt); + void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_double)(sqlite3_context*,double); + void (*result_error)(sqlite3_context*,const char*,int); + void (*result_error16)(sqlite3_context*,const void*,int); + void (*result_int)(sqlite3_context*,int); + void (*result_int64)(sqlite3_context*,sqlite_int64); + void (*result_null)(sqlite3_context*); + void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); + void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_value)(sqlite3_context*,sqlite3_value*); + void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,const char*,const char*),void*); + void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); + char * (*snprintf)(int,char*,const char*,...); + int (*step)(sqlite3_stmt*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,char const**,char const**,int*,int*,int*); + void (*thread_cleanup)(void); + int (*total_changes)(sqlite3*); + void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); + int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,sqlite_int64),void*); + void * (*user_data)(sqlite3_context*); + const void * (*value_blob)(sqlite3_value*); + int (*value_bytes)(sqlite3_value*); + int (*value_bytes16)(sqlite3_value*); + double (*value_double)(sqlite3_value*); + int (*value_int)(sqlite3_value*); + sqlite_int64 (*value_int64)(sqlite3_value*); + int (*value_numeric_type)(sqlite3_value*); + const unsigned char * (*value_text)(sqlite3_value*); + const void * (*value_text16)(sqlite3_value*); + const void * (*value_text16be)(sqlite3_value*); + const void * (*value_text16le)(sqlite3_value*); + int (*value_type)(sqlite3_value*); + char *(*vmprintf)(const char*,va_list); + /* Added ??? */ + int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); + /* Added by 3.3.13 */ + int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + int (*clear_bindings)(sqlite3_stmt*); + /* Added by 3.4.1 */ + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *)); + /* Added by 3.5.0 */ + int (*bind_zeroblob)(sqlite3_stmt*,int,int); + int (*blob_bytes)(sqlite3_blob*); + int (*blob_close)(sqlite3_blob*); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**); + int (*blob_read)(sqlite3_blob*,void*,int,int); + int (*blob_write)(sqlite3_blob*,const void*,int,int); + int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*)); + int (*file_control)(sqlite3*,const char*,int,void*); + sqlite3_int64 (*memory_highwater)(int); + sqlite3_int64 (*memory_used)(void); + sqlite3_mutex *(*mutex_alloc)(int); + void (*mutex_enter)(sqlite3_mutex*); + void (*mutex_free)(sqlite3_mutex*); + void (*mutex_leave)(sqlite3_mutex*); + int (*mutex_try)(sqlite3_mutex*); + int (*open_v2)(const char*,sqlite3**,int,const char*); + int (*release_memory)(int); + void (*result_error_nomem)(sqlite3_context*); + void (*result_error_toobig)(sqlite3_context*); + int (*sleep)(int); + void (*soft_heap_limit)(int); + sqlite3_vfs *(*vfs_find)(const char*); + int (*vfs_register)(sqlite3_vfs*,int); + int (*vfs_unregister)(sqlite3_vfs*); + int (*xthreadsafe)(void); + void (*result_zeroblob)(sqlite3_context*,int); + void (*result_error_code)(sqlite3_context*,int); + int (*test_control)(int, ...); + void (*randomness)(int,void*); + sqlite3 *(*context_db_handle)(sqlite3_context*); + int (*extended_result_codes)(sqlite3*,int); + int (*limit)(sqlite3*,int,int); + sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); + const char *(*sql)(sqlite3_stmt*); + int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); +}; - /* For operators other than UNION ALL we have to make sure that - ** the ORDER BY clause covers every term of the result set. Add - ** terms to the ORDER BY clause as necessary. - */ - if( op!=TK_ALL ){ - for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ - struct ExprList_item *pItem; - for(j=0, pItem=pOrderBy->a; jiCol>0 ); - if( pItem->iCol==i ) break; - } - if( j==nOrderBy ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->flags |= EP_IntValue; - pNew->u.iValue = i; - pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); - pOrderBy->a[nOrderBy++].iCol = (u16)i; - } - } - } - - /* Compute the comparison permutation and keyinfo that is used with - ** the permutation used to determine if the next - ** row of results comes from selectA or selectB. Also add explicit - ** collations to the ORDER BY clause terms so that when the subqueries - ** to the right and the left are evaluated, they use the correct - ** collation. - */ - aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); - if( aPermute ){ - struct ExprList_item *pItem; - for(i=0, pItem=pOrderBy->a; iiCol>0 && pItem->iCol<=p->pEList->nExpr ); - aPermute[i] = pItem->iCol - 1; - } - pKeyMerge = - sqlite3DbMallocRaw(db, sizeof(*pKeyMerge)+nOrderBy*(sizeof(CollSeq*)+1)); - if( pKeyMerge ){ - pKeyMerge->aSortOrder = (u8*)&pKeyMerge->aColl[nOrderBy]; - pKeyMerge->nField = (u16)nOrderBy; - pKeyMerge->enc = ENC(db); - for(i=0; ia[i].pExpr; - if( pTerm->flags & EP_ExpCollate ){ - pColl = pTerm->pColl; - }else{ - pColl = multiSelectCollSeq(pParse, p, aPermute[i]); - pTerm->flags |= EP_ExpCollate; - pTerm->pColl = pColl; - } - pKeyMerge->aColl[i] = pColl; - pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; - } - } - }else{ - pKeyMerge = 0; - } - - /* Reattach the ORDER BY clause to the query. - */ - p->pOrderBy = pOrderBy; - pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); +/* +** The following macros redefine the API routines so that they are +** redirected throught the global sqlite3_api structure. +** +** This header file is also used by the loadext.c source file +** (part of the main SQLite library - not an extension) so that +** it can get access to the sqlite3_api_routines structure +** definition. But the main library does not want to redefine +** the API. So the redefinition macros are only valid if the +** SQLITE_CORE macros is undefined. +*/ +#ifndef SQLITE_CORE +#define sqlite3_aggregate_context sqlite3_api->aggregate_context +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_aggregate_count sqlite3_api->aggregate_count +#endif +#define sqlite3_bind_blob sqlite3_api->bind_blob +#define sqlite3_bind_double sqlite3_api->bind_double +#define sqlite3_bind_int sqlite3_api->bind_int +#define sqlite3_bind_int64 sqlite3_api->bind_int64 +#define sqlite3_bind_null sqlite3_api->bind_null +#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count +#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index +#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name +#define sqlite3_bind_text sqlite3_api->bind_text +#define sqlite3_bind_text16 sqlite3_api->bind_text16 +#define sqlite3_bind_value sqlite3_api->bind_value +#define sqlite3_busy_handler sqlite3_api->busy_handler +#define sqlite3_busy_timeout sqlite3_api->busy_timeout +#define sqlite3_changes sqlite3_api->changes +#define sqlite3_close sqlite3_api->close +#define sqlite3_collation_needed sqlite3_api->collation_needed +#define sqlite3_collation_needed16 sqlite3_api->collation_needed16 +#define sqlite3_column_blob sqlite3_api->column_blob +#define sqlite3_column_bytes sqlite3_api->column_bytes +#define sqlite3_column_bytes16 sqlite3_api->column_bytes16 +#define sqlite3_column_count sqlite3_api->column_count +#define sqlite3_column_database_name sqlite3_api->column_database_name +#define sqlite3_column_database_name16 sqlite3_api->column_database_name16 +#define sqlite3_column_decltype sqlite3_api->column_decltype +#define sqlite3_column_decltype16 sqlite3_api->column_decltype16 +#define sqlite3_column_double sqlite3_api->column_double +#define sqlite3_column_int sqlite3_api->column_int +#define sqlite3_column_int64 sqlite3_api->column_int64 +#define sqlite3_column_name sqlite3_api->column_name +#define sqlite3_column_name16 sqlite3_api->column_name16 +#define sqlite3_column_origin_name sqlite3_api->column_origin_name +#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 +#define sqlite3_column_table_name sqlite3_api->column_table_name +#define sqlite3_column_table_name16 sqlite3_api->column_table_name16 +#define sqlite3_column_text sqlite3_api->column_text +#define sqlite3_column_text16 sqlite3_api->column_text16 +#define sqlite3_column_type sqlite3_api->column_type +#define sqlite3_column_value sqlite3_api->column_value +#define sqlite3_commit_hook sqlite3_api->commit_hook +#define sqlite3_complete sqlite3_api->complete +#define sqlite3_complete16 sqlite3_api->complete16 +#define sqlite3_create_collation sqlite3_api->create_collation +#define sqlite3_create_collation16 sqlite3_api->create_collation16 +#define sqlite3_create_function sqlite3_api->create_function +#define sqlite3_create_function16 sqlite3_api->create_function16 +#define sqlite3_create_module sqlite3_api->create_module +#define sqlite3_create_module_v2 sqlite3_api->create_module_v2 +#define sqlite3_data_count sqlite3_api->data_count +#define sqlite3_db_handle sqlite3_api->db_handle +#define sqlite3_declare_vtab sqlite3_api->declare_vtab +#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache +#define sqlite3_errcode sqlite3_api->errcode +#define sqlite3_errmsg sqlite3_api->errmsg +#define sqlite3_errmsg16 sqlite3_api->errmsg16 +#define sqlite3_exec sqlite3_api->exec +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_expired sqlite3_api->expired +#endif +#define sqlite3_finalize sqlite3_api->finalize +#define sqlite3_free sqlite3_api->free +#define sqlite3_free_table sqlite3_api->free_table +#define sqlite3_get_autocommit sqlite3_api->get_autocommit +#define sqlite3_get_auxdata sqlite3_api->get_auxdata +#define sqlite3_get_table sqlite3_api->get_table +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_global_recover sqlite3_api->global_recover +#endif +#define sqlite3_interrupt sqlite3_api->interruptx +#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid +#define sqlite3_libversion sqlite3_api->libversion +#define sqlite3_libversion_number sqlite3_api->libversion_number +#define sqlite3_malloc sqlite3_api->malloc +#define sqlite3_mprintf sqlite3_api->mprintf +#define sqlite3_open sqlite3_api->open +#define sqlite3_open16 sqlite3_api->open16 +#define sqlite3_prepare sqlite3_api->prepare +#define sqlite3_prepare16 sqlite3_api->prepare16 +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_profile sqlite3_api->profile +#define sqlite3_progress_handler sqlite3_api->progress_handler +#define sqlite3_realloc sqlite3_api->realloc +#define sqlite3_reset sqlite3_api->reset +#define sqlite3_result_blob sqlite3_api->result_blob +#define sqlite3_result_double sqlite3_api->result_double +#define sqlite3_result_error sqlite3_api->result_error +#define sqlite3_result_error16 sqlite3_api->result_error16 +#define sqlite3_result_int sqlite3_api->result_int +#define sqlite3_result_int64 sqlite3_api->result_int64 +#define sqlite3_result_null sqlite3_api->result_null +#define sqlite3_result_text sqlite3_api->result_text +#define sqlite3_result_text16 sqlite3_api->result_text16 +#define sqlite3_result_text16be sqlite3_api->result_text16be +#define sqlite3_result_text16le sqlite3_api->result_text16le +#define sqlite3_result_value sqlite3_api->result_value +#define sqlite3_rollback_hook sqlite3_api->rollback_hook +#define sqlite3_set_authorizer sqlite3_api->set_authorizer +#define sqlite3_set_auxdata sqlite3_api->set_auxdata +#define sqlite3_snprintf sqlite3_api->snprintf +#define sqlite3_step sqlite3_api->step +#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata +#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup +#define sqlite3_total_changes sqlite3_api->total_changes +#define sqlite3_trace sqlite3_api->trace +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings +#endif +#define sqlite3_update_hook sqlite3_api->update_hook +#define sqlite3_user_data sqlite3_api->user_data +#define sqlite3_value_blob sqlite3_api->value_blob +#define sqlite3_value_bytes sqlite3_api->value_bytes +#define sqlite3_value_bytes16 sqlite3_api->value_bytes16 +#define sqlite3_value_double sqlite3_api->value_double +#define sqlite3_value_int sqlite3_api->value_int +#define sqlite3_value_int64 sqlite3_api->value_int64 +#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type +#define sqlite3_value_text sqlite3_api->value_text +#define sqlite3_value_text16 sqlite3_api->value_text16 +#define sqlite3_value_text16be sqlite3_api->value_text16be +#define sqlite3_value_text16le sqlite3_api->value_text16le +#define sqlite3_value_type sqlite3_api->value_type +#define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_overload_function sqlite3_api->overload_function +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_clear_bindings sqlite3_api->clear_bindings +#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob +#define sqlite3_blob_bytes sqlite3_api->blob_bytes +#define sqlite3_blob_close sqlite3_api->blob_close +#define sqlite3_blob_open sqlite3_api->blob_open +#define sqlite3_blob_read sqlite3_api->blob_read +#define sqlite3_blob_write sqlite3_api->blob_write +#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 +#define sqlite3_file_control sqlite3_api->file_control +#define sqlite3_memory_highwater sqlite3_api->memory_highwater +#define sqlite3_memory_used sqlite3_api->memory_used +#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc +#define sqlite3_mutex_enter sqlite3_api->mutex_enter +#define sqlite3_mutex_free sqlite3_api->mutex_free +#define sqlite3_mutex_leave sqlite3_api->mutex_leave +#define sqlite3_mutex_try sqlite3_api->mutex_try +#define sqlite3_open_v2 sqlite3_api->open_v2 +#define sqlite3_release_memory sqlite3_api->release_memory +#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem +#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig +#define sqlite3_sleep sqlite3_api->sleep +#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit +#define sqlite3_vfs_find sqlite3_api->vfs_find +#define sqlite3_vfs_register sqlite3_api->vfs_register +#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister +#define sqlite3_threadsafe sqlite3_api->xthreadsafe +#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob +#define sqlite3_result_error_code sqlite3_api->result_error_code +#define sqlite3_test_control sqlite3_api->test_control +#define sqlite3_randomness sqlite3_api->randomness +#define sqlite3_context_db_handle sqlite3_api->context_db_handle +#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes +#define sqlite3_limit sqlite3_api->limit +#define sqlite3_next_stmt sqlite3_api->next_stmt +#define sqlite3_sql sqlite3_api->sql +#define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook +#endif /* SQLITE_CORE */ - /* Allocate a range of temporary registers and the KeyInfo needed - ** for the logic that removes duplicate result rows when the - ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). - */ - if( op==TK_ALL ){ - regPrev = 0; - }else{ - int nExpr = p->pEList->nExpr; - assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite3GetTempRange(pParse, nExpr+1); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); - pKeyDup = sqlite3DbMallocZero(db, - sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); - if( pKeyDup ){ - pKeyDup->aSortOrder = (u8*)&pKeyDup->aColl[nExpr]; - pKeyDup->nField = (u16)nExpr; - pKeyDup->enc = ENC(db); - for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); - pKeyDup->aSortOrder[i] = 0; - } - } - } - - /* Separate the left and the right query from one another - */ - p->pPrior = 0; - sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); - if( pPrior->pPrior==0 ){ - sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); - } +#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; +#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; - /* Compute the limit registers */ - computeLimitRegisters(pParse, p, labelEnd); - if( p->iLimit && op==TK_ALL ){ - regLimitA = ++pParse->nMem; - regLimitB = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, - regLimitA); - sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB); - }else{ - regLimitA = regLimitB = 0; - } - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = 0; - sqlite3ExprDelete(db, p->pOffset); - p->pOffset = 0; +#endif /* _SQLITE3EXT_H_ */ - regAddrA = ++pParse->nMem; - regEofA = ++pParse->nMem; - regAddrB = ++pParse->nMem; - regEofB = ++pParse->nMem; - regOutA = ++pParse->nMem; - regOutB = ++pParse->nMem; - sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); - sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); +/************** End of sqlite3ext.h ******************************************/ +/************** Continuing where we left off in loadext.c ********************/ - /* Jump past the various subroutines and coroutines to the main - ** merge loop - */ - j1 = sqlite3VdbeAddOp0(v, OP_Goto); - addrSelectA = sqlite3VdbeCurrentAddr(v); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Some API routines are omitted when various features are +** excluded from a build of SQLite. Substitute a NULL pointer +** for any missing APIs. +*/ +#ifndef SQLITE_ENABLE_COLUMN_METADATA +# define sqlite3_column_database_name 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name 0 +# define sqlite3_column_origin_name16 0 +# define sqlite3_table_column_metadata 0 +#endif - /* Generate a coroutine to evaluate the SELECT statement to the - ** left of the compound operator - the "A" select. - */ - VdbeNoopComment((v, "Begin coroutine for left SELECT")); - pPrior->iLimit = regLimitA; - explainSetInteger(iSub1, pParse->iNextSelectId); - sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - VdbeNoopComment((v, "End coroutine for left SELECT")); +#ifdef SQLITE_OMIT_AUTHORIZATION +# define sqlite3_set_authorizer 0 +#endif - /* Generate a coroutine to evaluate the SELECT statement on - ** the right - the "B" select - */ - addrSelectB = sqlite3VdbeCurrentAddr(v); - VdbeNoopComment((v, "Begin coroutine for right SELECT")); - savedLimit = p->iLimit; - savedOffset = p->iOffset; - p->iLimit = regLimitB; - p->iOffset = 0; - explainSetInteger(iSub2, pParse->iNextSelectId); - sqlite3Select(pParse, p, &destB); - p->iLimit = savedLimit; - p->iOffset = savedOffset; - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofB); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - VdbeNoopComment((v, "End coroutine for right SELECT")); +#ifdef SQLITE_OMIT_UTF16 +# define sqlite3_bind_text16 0 +# define sqlite3_collation_needed16 0 +# define sqlite3_column_decltype16 0 +# define sqlite3_column_name16 0 +# define sqlite3_column_text16 0 +# define sqlite3_complete16 0 +# define sqlite3_create_collation16 0 +# define sqlite3_create_function16 0 +# define sqlite3_errmsg16 0 +# define sqlite3_open16 0 +# define sqlite3_prepare16 0 +# define sqlite3_prepare16_v2 0 +# define sqlite3_result_error16 0 +# define sqlite3_result_text16 0 +# define sqlite3_result_text16be 0 +# define sqlite3_result_text16le 0 +# define sqlite3_value_text16 0 +# define sqlite3_value_text16be 0 +# define sqlite3_value_text16le 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name16 0 +#endif - /* Generate a subroutine that outputs the current row of the A - ** select as the next output row of the compound select. - */ - VdbeNoopComment((v, "Output routine for A")); - addrOutA = generateOutputSubroutine(pParse, - p, &destA, pDest, regOutA, - regPrev, pKeyDup, P4_KEYINFO_HANDOFF, labelEnd); - - /* Generate a subroutine that outputs the current row of the B - ** select as the next output row of the compound select. - */ - if( op==TK_ALL || op==TK_UNION ){ - VdbeNoopComment((v, "Output routine for B")); - addrOutB = generateOutputSubroutine(pParse, - p, &destB, pDest, regOutB, - regPrev, pKeyDup, P4_KEYINFO_STATIC, labelEnd); - } +#ifdef SQLITE_OMIT_COMPLETE +# define sqlite3_complete 0 +# define sqlite3_complete16 0 +#endif - /* Generate a subroutine to run when the results from select A - ** are exhausted and only data in select B remains. - */ - VdbeNoopComment((v, "eof-A subroutine")); - if( op==TK_EXCEPT || op==TK_INTERSECT ){ - addrEofA = sqlite3VdbeAddOp2(v, OP_Goto, 0, labelEnd); - }else{ - addrEofA = sqlite3VdbeAddOp2(v, OP_If, regEofB, labelEnd); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); - p->nSelectRow += pPrior->nSelectRow; - } +#ifdef SQLITE_OMIT_DECLTYPE +# define sqlite3_column_decltype16 0 +# define sqlite3_column_decltype 0 +#endif - /* Generate a subroutine to run when the results from select B - ** are exhausted and only data in select A remains. - */ - if( op==TK_INTERSECT ){ - addrEofB = addrEofA; - if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; - }else{ - VdbeNoopComment((v, "eof-B subroutine")); - addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); - } +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK +# define sqlite3_progress_handler 0 +#endif - /* Generate code to handle the case of AB - */ - VdbeNoopComment((v, "A-gt-B subroutine")); - addrAgtB = sqlite3VdbeCurrentAddr(v); - if( op==TK_ALL || op==TK_UNION ){ - sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); - } - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); +#ifdef SQLITE_OMIT_TRACE +# define sqlite3_profile 0 +# define sqlite3_trace 0 +#endif - /* This code runs once to initialize everything. - */ - sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofA); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofB); - sqlite3VdbeAddOp2(v, OP_Gosub, regAddrA, addrSelectA); - sqlite3VdbeAddOp2(v, OP_Gosub, regAddrB, addrSelectB); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); - sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); +#ifdef SQLITE_OMIT_GET_TABLE +# define sqlite3_free_table 0 +# define sqlite3_get_table 0 +#endif - /* Implement the main merge loop - */ - sqlite3VdbeResolveLabel(v, labelCmpr); - sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); - sqlite3VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, - (char*)pKeyMerge, P4_KEYINFO_HANDOFF); - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); +#ifdef SQLITE_OMIT_INCRBLOB +#define sqlite3_bind_zeroblob 0 +#define sqlite3_blob_bytes 0 +#define sqlite3_blob_close 0 +#define sqlite3_blob_open 0 +#define sqlite3_blob_read 0 +#define sqlite3_blob_write 0 +#endif - /* Release temporary registers - */ - if( regPrev ){ - sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - - /* Jump to the this point in order to terminate the query. +/* +** The following structure contains pointers to all SQLite API routines. +** A pointer to this structure is passed into extensions when they are +** loaded so that the extension can make calls back into the SQLite +** library. +** +** When adding new APIs, add them to the bottom of this structure +** in order to preserve backwards compatibility. +** +** Extensions that use newer APIs should first call the +** sqlite3_libversion_number() to make sure that the API they +** intend to use is supported by the library. Extensions should +** also check to make sure that the pointer to the function is +** not NULL before calling it. +*/ +static const sqlite3_api_routines sqlite3Apis = { + sqlite3_aggregate_context, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_aggregate_count, +#else + 0, +#endif + sqlite3_bind_blob, + sqlite3_bind_double, + sqlite3_bind_int, + sqlite3_bind_int64, + sqlite3_bind_null, + sqlite3_bind_parameter_count, + sqlite3_bind_parameter_index, + sqlite3_bind_parameter_name, + sqlite3_bind_text, + sqlite3_bind_text16, + sqlite3_bind_value, + sqlite3_busy_handler, + sqlite3_busy_timeout, + sqlite3_changes, + sqlite3_close, + sqlite3_collation_needed, + sqlite3_collation_needed16, + sqlite3_column_blob, + sqlite3_column_bytes, + sqlite3_column_bytes16, + sqlite3_column_count, + sqlite3_column_database_name, + sqlite3_column_database_name16, + sqlite3_column_decltype, + sqlite3_column_decltype16, + sqlite3_column_double, + sqlite3_column_int, + sqlite3_column_int64, + sqlite3_column_name, + sqlite3_column_name16, + sqlite3_column_origin_name, + sqlite3_column_origin_name16, + sqlite3_column_table_name, + sqlite3_column_table_name16, + sqlite3_column_text, + sqlite3_column_text16, + sqlite3_column_type, + sqlite3_column_value, + sqlite3_commit_hook, + sqlite3_complete, + sqlite3_complete16, + sqlite3_create_collation, + sqlite3_create_collation16, + sqlite3_create_function, + sqlite3_create_function16, + sqlite3_create_module, + sqlite3_data_count, + sqlite3_db_handle, + sqlite3_declare_vtab, + sqlite3_enable_shared_cache, + sqlite3_errcode, + sqlite3_errmsg, + sqlite3_errmsg16, + sqlite3_exec, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_expired, +#else + 0, +#endif + sqlite3_finalize, + sqlite3_free, + sqlite3_free_table, + sqlite3_get_autocommit, + sqlite3_get_auxdata, + sqlite3_get_table, + 0, /* Was sqlite3_global_recover(), but that function is deprecated */ + sqlite3_interrupt, + sqlite3_last_insert_rowid, + sqlite3_libversion, + sqlite3_libversion_number, + sqlite3_malloc, + sqlite3_mprintf, + sqlite3_open, + sqlite3_open16, + sqlite3_prepare, + sqlite3_prepare16, + sqlite3_profile, + sqlite3_progress_handler, + sqlite3_realloc, + sqlite3_reset, + sqlite3_result_blob, + sqlite3_result_double, + sqlite3_result_error, + sqlite3_result_error16, + sqlite3_result_int, + sqlite3_result_int64, + sqlite3_result_null, + sqlite3_result_text, + sqlite3_result_text16, + sqlite3_result_text16be, + sqlite3_result_text16le, + sqlite3_result_value, + sqlite3_rollback_hook, + sqlite3_set_authorizer, + sqlite3_set_auxdata, + sqlite3_snprintf, + sqlite3_step, + sqlite3_table_column_metadata, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_thread_cleanup, +#else + 0, +#endif + sqlite3_total_changes, + sqlite3_trace, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_transfer_bindings, +#else + 0, +#endif + sqlite3_update_hook, + sqlite3_user_data, + sqlite3_value_blob, + sqlite3_value_bytes, + sqlite3_value_bytes16, + sqlite3_value_double, + sqlite3_value_int, + sqlite3_value_int64, + sqlite3_value_numeric_type, + sqlite3_value_text, + sqlite3_value_text16, + sqlite3_value_text16be, + sqlite3_value_text16le, + sqlite3_value_type, + sqlite3_vmprintf, + /* + ** The original API set ends here. All extensions can call any + ** of the APIs above provided that the pointer is not NULL. But + ** before calling APIs that follow, extension should check the + ** sqlite3_libversion_number() to make sure they are dealing with + ** a library that is new enough to support that API. + ************************************************************************* */ - sqlite3VdbeResolveLabel(v, labelEnd); + sqlite3_overload_function, - /* Set the number of output columns + /* + ** Added after 3.3.13 */ - if( pDest->eDest==SRT_Output ){ - Select *pFirst = pPrior; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); - } + sqlite3_prepare_v2, + sqlite3_prepare16_v2, + sqlite3_clear_bindings, - /* Reassembly the compound query so that it will be freed correctly - ** by the calling function */ - if( p->pPrior ){ - sqlite3SelectDelete(db, p->pPrior); - } - p->pPrior = pPrior; + /* + ** Added for 3.4.1 + */ + sqlite3_create_module_v2, - /*** TBD: Insert subroutine calls to close cursors on incomplete - **** subqueries ****/ - explainComposite(pParse, p->op, iSub1, iSub2, 0); - return SQLITE_OK; -} + /* + ** Added for 3.5.0 + */ + sqlite3_bind_zeroblob, + sqlite3_blob_bytes, + sqlite3_blob_close, + sqlite3_blob_open, + sqlite3_blob_read, + sqlite3_blob_write, + sqlite3_create_collation_v2, + sqlite3_file_control, + sqlite3_memory_highwater, + sqlite3_memory_used, +#ifdef SQLITE_MUTEX_OMIT + 0, + 0, + 0, + 0, + 0, +#else + sqlite3_mutex_alloc, + sqlite3_mutex_enter, + sqlite3_mutex_free, + sqlite3_mutex_leave, + sqlite3_mutex_try, #endif + sqlite3_open_v2, + sqlite3_release_memory, + sqlite3_result_error_nomem, + sqlite3_result_error_toobig, + sqlite3_sleep, + sqlite3_soft_heap_limit, + sqlite3_vfs_find, + sqlite3_vfs_register, + sqlite3_vfs_unregister, -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) -/* Forward Declarations */ -static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); + /* + ** Added for 3.5.8 + */ + sqlite3_threadsafe, + sqlite3_result_zeroblob, + sqlite3_result_error_code, + sqlite3_test_control, + sqlite3_randomness, + sqlite3_context_db_handle, + + /* + ** Added for 3.6.0 + */ + sqlite3_extended_result_codes, + sqlite3_limit, + sqlite3_next_stmt, + sqlite3_sql, + sqlite3_status, + + /* + ** Added for 3.7.4 + */ + sqlite3_backup_finish, + sqlite3_backup_init, + sqlite3_backup_pagecount, + sqlite3_backup_remaining, + sqlite3_backup_step, +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + sqlite3_compileoption_get, + sqlite3_compileoption_used, +#else + 0, + 0, +#endif + sqlite3_create_function_v2, + sqlite3_db_config, + sqlite3_db_mutex, + sqlite3_db_status, + sqlite3_extended_errcode, + sqlite3_log, + sqlite3_soft_heap_limit64, + sqlite3_sourceid, + sqlite3_stmt_status, + sqlite3_strnicmp, +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + sqlite3_unlock_notify, +#else + 0, +#endif +#ifndef SQLITE_OMIT_WAL + sqlite3_wal_autocheckpoint, + sqlite3_wal_checkpoint, + sqlite3_wal_hook, +#else + 0, + 0, + 0, +#endif +}; /* -** Scan through the expression pExpr. Replace every reference to -** a column in table number iTable with a copy of the iColumn-th -** entry in pEList. (But leave references to the ROWID column -** unchanged.) +** Attempt to load an SQLite extension library contained in the file +** zFile. The entry point is zProc. zProc may be 0 in which case a +** default entry point name (sqlite3_extension_init) is used. Use +** of the default name is recommended. ** -** This routine is part of the flattening procedure. A subquery -** whose result set is defined by pEList appears as entry in the -** FROM clause of a SELECT such that the VDBE cursor assigned to that -** FORM clause entry is iTable. This routine make the necessary -** changes to pExpr so that it refers directly to the source table -** of the subquery rather the result set of the subquery. +** Return SQLITE_OK on success and SQLITE_ERROR if something goes wrong. +** +** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with +** error message text. The calling function should free this memory +** by calling sqlite3DbFree(db, ). */ -static Expr *substExpr( - sqlite3 *db, /* Report malloc errors to this connection */ - Expr *pExpr, /* Expr in which substitution occurs */ - int iTable, /* Table to be substituted */ - ExprList *pEList /* Substitute expressions */ +static int sqlite3LoadExtension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ ){ - if( pExpr==0 ) return 0; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ - if( pExpr->iColumn<0 ){ - pExpr->op = TK_NULL; - }else{ - Expr *pNew; - assert( pEList!=0 && pExpr->iColumnnExpr ); - assert( pExpr->pLeft==0 && pExpr->pRight==0 ); - pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - if( pNew && pExpr->pColl ){ - pNew->pColl = pExpr->pColl; + sqlite3_vfs *pVfs = db->pVfs; + void *handle; + int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + char *zErrmsg = 0; + void **aHandle; + const int nMsg = 300; + + if( pzErrMsg ) *pzErrMsg = 0; + + /* Ticket #1863. To avoid a creating security problems for older + ** applications that relink against newer versions of SQLite, the + ** ability to run load_extension is turned off by default. One + ** must call sqlite3_enable_load_extension() to turn on extension + ** loading. Otherwise you get the following error. + */ + if( (db->flags & SQLITE_LoadExtension)==0 ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("not authorized"); + } + return SQLITE_ERROR; + } + + if( zProc==0 ){ + zProc = "sqlite3_extension_init"; + } + + handle = sqlite3OsDlOpen(pVfs, zFile); + if( handle==0 ){ + if( pzErrMsg ){ + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + if( zErrmsg ){ + sqlite3_snprintf(nMsg, zErrmsg, + "unable to open shared library [%s]", zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3ExprDelete(db, pExpr); - pExpr = pNew; } - }else{ - pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); - pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); - }else{ - substExprList(db, pExpr->x.pList, iTable, pEList); + return SQLITE_ERROR; + } + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zProc); + if( xInit==0 ){ + if( pzErrMsg ){ + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); + if( zErrmsg ){ + sqlite3_snprintf(nMsg, zErrmsg, + "no entry point [%s] in shared library [%s]", zProc,zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); + } + sqlite3OsDlClose(pVfs, handle); + } + return SQLITE_ERROR; + }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } + sqlite3_free(zErrmsg); + sqlite3OsDlClose(pVfs, handle); + return SQLITE_ERROR; } - return pExpr; + + /* Append the new shared library handle to the db->aExtension array. */ + aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); + if( aHandle==0 ){ + return SQLITE_NOMEM; + } + if( db->nExtension>0 ){ + memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); + } + sqlite3DbFree(db, db->aExtension); + db->aExtension = aHandle; + + db->aExtension[db->nExtension++] = handle; + return SQLITE_OK; } -static void substExprList( - sqlite3 *db, /* Report malloc errors here */ - ExprList *pList, /* List to scan and in which to make substitutes */ - int iTable, /* Table to be substituted */ - ExprList *pEList /* Substitute values */ +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ ){ + int rc; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Call this routine when the database connection is closing in order +** to clean up loaded extensions +*/ +SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ int i; - if( pList==0 ) return; - for(i=0; inExpr; i++){ - pList->a[i].pExpr = substExpr(db, pList->a[i].pExpr, iTable, pEList); + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inExtension; i++){ + sqlite3OsDlClose(db->pVfs, db->aExtension[i]); } + sqlite3DbFree(db, db->aExtension); } -static void substSelect( - sqlite3 *db, /* Report malloc errors here */ - Select *p, /* SELECT statement in which to make substitutions */ - int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ -){ - SrcList *pSrc; - struct SrcList_item *pItem; - int i; - if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ - for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); - } + +/* +** Enable or disable extension loading. Extension loading is disabled by +** default so as not to open security holes in older applications. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); + if( onoff ){ + db->flags |= SQLITE_LoadExtension; + }else{ + db->flags &= ~SQLITE_LoadExtension; } + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } -#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + /* -** This routine attempts to flatten subqueries in order to speed -** execution. It returns 1 if it makes changes and 0 if no flattening -** occurs. -** -** To understand the concept of flattening, consider the following -** query: -** -** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5 -** -** The default way of implementing this query is to execute the -** subquery first and store the results in a temporary table, then -** run the outer query on that temporary table. This requires two -** passes over the data. Furthermore, because the temporary table -** has no indices, the WHERE clause on the outer query cannot be -** optimized. -** -** This routine attempts to rewrite queries such as the above into -** a single flat select, like this: -** -** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 -** -** The code generated for this simpification gives the same result -** but only has to scan the data once. And because indices might -** exist on the table t1, a complete scan of the data might be -** avoided. -** -** Flattening is only attempted if all of the following are true: +** The auto-extension code added regardless of whether or not extension +** loading is supported. We need a dummy sqlite3Apis pointer for that +** code if regular extension loading is not available. This is that +** dummy pointer. +*/ +#ifdef SQLITE_OMIT_LOAD_EXTENSION +static const sqlite3_api_routines sqlite3Apis = { 0 }; +#endif + + +/* +** The following object holds the list of automatically loaded +** extensions. ** -** (1) The subquery and the outer query do not both use aggregates. -** -** (2) The subquery is not an aggregate or the outer query is not a join. -** -** (3) The subquery is not the right operand of a left outer join -** (Originally ticket #306. Strengthened by ticket #3300) -** -** (4) The subquery is not DISTINCT. -** -** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT -** sub-queries that were excluded from this optimization. Restriction -** (4) has since been expanded to exclude all DISTINCT subqueries. -** -** (6) The subquery does not use aggregates or the outer query is not -** DISTINCT. -** -** (7) The subquery has a FROM clause. -** -** (8) The subquery does not use LIMIT or the outer query is not a join. -** -** (9) The subquery does not use LIMIT or the outer query does not use -** aggregates. -** -** (10) The subquery does not use aggregates or the outer query does not -** use LIMIT. -** -** (11) The subquery and the outer query do not both have ORDER BY clauses. -** -** (**) Not implemented. Subsumed into restriction (3). Was previously -** a separate restriction deriving from ticket #350. -** -** (13) The subquery and outer query do not both use LIMIT. -** -** (14) The subquery does not use OFFSET. -** -** (15) The outer query is not part of a compound select or the -** subquery does not have a LIMIT clause. -** (See ticket #2339 and ticket [02a8e81d44]). -** -** (16) The outer query is not an aggregate or the subquery does -** not contain ORDER BY. (Ticket #2942) This used to not matter -** until we introduced the group_concat() function. -** -** (17) The sub-query is not a compound select, or it is a UNION ALL -** compound clause made up entirely of non-aggregate queries, and -** the parent query: -** -** * is not itself part of a compound select, -** * is not an aggregate or DISTINCT query, and -** * has no other tables or sub-selects in the FROM clause. -** -** The parent and sub-query may contain WHERE clauses. Subject to -** rules (11), (13) and (14), they may also contain ORDER BY, -** LIMIT and OFFSET clauses. -** -** (18) If the sub-query is a compound select, then all terms of the -** ORDER by clause of the parent must be simple references to -** columns of the sub-query. -** -** (19) The subquery does not use LIMIT or the outer query does not -** have a WHERE clause. -** -** (20) If the sub-query is a compound select, then it must not use -** an ORDER BY clause. Ticket #3773. We could relax this constraint -** somewhat by saying that the terms of the ORDER BY clause must -** appear as unmodified result columns in the outer query. But -** have other optimizations in mind to deal with that case. -** -** In this routine, the "p" parameter is a pointer to the outer query. -** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query -** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. -** -** If flattening is not attempted, this routine is a no-op and returns 0. -** If flattening is attempted this routine returns 1. -** -** All of the expression analysis must occur on both the outer query and -** the subquery before this routine runs. +** This list is shared across threads. The SQLITE_MUTEX_STATIC_MASTER +** mutex must be held while accessing this list. */ -static int flattenSubquery( - Parse *pParse, /* Parsing context */ - Select *p, /* The parent or outer SELECT statement */ - int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ - int isAgg, /* True if outer SELECT uses aggregate functions */ - int subqueryIsAgg /* True if the subquery uses aggregate functions */ -){ - const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; - Select *pSub; /* The inner query or "subquery" */ - Select *pSub1; /* Pointer to the rightmost select in sub-query */ - SrcList *pSrc; /* The FROM clause of the outer query */ - SrcList *pSubSrc; /* The FROM clause of the subquery */ - ExprList *pList; /* The result set of the outer query */ - int iParent; /* VDBE cursor number of the pSub result set temp table */ - int i; /* Loop counter */ - Expr *pWhere; /* The WHERE clause */ - struct SrcList_item *pSubitem; /* The subquery */ - sqlite3 *db = pParse->db; +typedef struct sqlite3AutoExtList sqlite3AutoExtList; +static SQLITE_WSD struct sqlite3AutoExtList { + int nExt; /* Number of entries in aExt[] */ + void (**aExt)(void); /* Pointers to the extension init functions */ +} sqlite3Autoext = { 0, 0 }; - /* Check to see if flattening is permitted. Return 0 if not. - */ - assert( p!=0 ); - assert( p->pPrior==0 ); /* Unable to flatten compound queries */ - if( db->flags & SQLITE_QueryFlattener ) return 0; - pSrc = p->pSrc; - assert( pSrc && iFrom>=0 && iFromnSrc ); - pSubitem = &pSrc->a[iFrom]; - iParent = pSubitem->iCursor; - pSub = pSubitem->pSelect; - assert( pSub!=0 ); - if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ - if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ - pSubSrc = pSub->pSrc; - assert( pSubSrc ); - /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, - ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET - ** because they could be computed at compile-time. But when LIMIT and OFFSET - ** became arbitrary expressions, we were forced to add restrictions (13) - ** and (14). */ - if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ - if( pSub->pOffset ) return 0; /* Restriction (14) */ - if( p->pRightmost && pSub->pLimit ){ - return 0; /* Restriction (15) */ - } - if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ - if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ - if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ - return 0; /* Restrictions (8)(9) */ - } - if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ - return 0; /* Restriction (6) */ - } - if( p->pOrderBy && pSub->pOrderBy ){ - return 0; /* Restriction (11) */ - } - if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ - if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ +/* The "wsdAutoext" macro will resolve to the autoextension +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Autoext" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdAutoextInit \ + sqlite3AutoExtList *x = &GLOBAL(sqlite3AutoExtList,sqlite3Autoext) +# define wsdAutoext x[0] +#else +# define wsdAutoextInit +# define wsdAutoext sqlite3Autoext +#endif - /* OBSOLETE COMMENT 1: - ** Restriction 3: If the subquery is a join, make sure the subquery is - ** not used as the right operand of an outer join. Examples of why this - ** is not allowed: - ** - ** t1 LEFT OUTER JOIN (t2 JOIN t3) - ** - ** If we flatten the above, we would get - ** - ** (t1 LEFT OUTER JOIN t2) JOIN t3 - ** - ** which is not at all the same thing. - ** - ** OBSOLETE COMMENT 2: - ** Restriction 12: If the subquery is the right operand of a left outer - ** join, make sure the subquery has no WHERE clause. - ** An examples of why this is not allowed: - ** - ** t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0) - ** - ** If we flatten the above, we would get - ** - ** (t1 LEFT OUTER JOIN t2) WHERE t2.x>0 - ** - ** But the t2.x>0 test will always fail on a NULL row of t2, which - ** effectively converts the OUTER JOIN into an INNER JOIN. - ** - ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE: - ** Ticket #3300 shows that flattening the right term of a LEFT JOIN - ** is fraught with danger. Best to avoid the whole thing. If the - ** subquery is the right term of a LEFT JOIN, then do not flatten. - */ - if( (pSubitem->jointype & JT_OUTER)!=0 ){ - return 0; - } - /* Restriction 17: If the sub-query is a compound SELECT, then it must - ** use only the UNION ALL operator. And none of the simple select queries - ** that make up the compound SELECT are allowed to be aggregate or distinct - ** queries. - */ - if( pSub->pPrior ){ - if( pSub->pOrderBy ){ - return 0; /* Restriction 20 */ - } - if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){ - return 0; - } - for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ - testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); - testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); - if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 - || (pSub1->pPrior && pSub1->op!=TK_ALL) - || NEVER(pSub1->pSrc==0) || pSub1->pSrc->nSrc!=1 - ){ - return 0; - } +/* +** Register a statically linked extension that is automatically +** loaded by every new database connection. +*/ +SQLITE_API int sqlite3_auto_extension(void (*xInit)(void)){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ){ + return rc; + }else +#endif + { + int i; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + for(i=0; ipOrderBy ){ - int ii; - for(ii=0; iipOrderBy->nExpr; ii++){ - if( p->pOrderBy->a[ii].iCol==0 ) return 0; + if( i==wsdAutoext.nExt ){ + int nByte = (wsdAutoext.nExt+1)*sizeof(wsdAutoext.aExt[0]); + void (**aNew)(void); + aNew = sqlite3_realloc(wsdAutoext.aExt, nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + wsdAutoext.aExt = aNew; + wsdAutoext.aExt[wsdAutoext.nExt] = xInit; + wsdAutoext.nExt++; } } + sqlite3_mutex_leave(mutex); + assert( (rc&0xff)==rc ); + return rc; } +} - /***** If we reach this point, flattening is permitted. *****/ +/* +** Reset the automatic extension loading mechanism. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize()==SQLITE_OK ) +#endif + { +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + sqlite3_free(wsdAutoext.aExt); + wsdAutoext.aExt = 0; + wsdAutoext.nExt = 0; + sqlite3_mutex_leave(mutex); + } +} - /* Authorize the subquery */ - pParse->zAuthContext = pSubitem->zName; - sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); - pParse->zAuthContext = zSavedAuthContext; +/* +** Load all automatic extensions. +** +** If anything goes wrong, set an error in the database connection. +*/ +SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ + int i; + int go = 1; + int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); - /* If the sub-query is a compound SELECT statement, then (by restrictions - ** 17 and 18 above) it must be a UNION ALL and the parent query must - ** be of the form: - ** - ** SELECT FROM () - ** - ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block - ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or - ** OFFSET clauses and joins them to the left-hand-side of the original - ** using UNION ALL operators. In this case N is the number of simple - ** select statements in the compound sub-query. - ** - ** Example: - ** - ** SELECT a+1 FROM ( - ** SELECT x FROM tab - ** UNION ALL - ** SELECT y FROM tab - ** UNION ALL - ** SELECT abs(z*2) FROM tab2 - ** ) WHERE a!=5 ORDER BY 1 - ** - ** Transformed into: - ** - ** SELECT x+1 FROM tab WHERE x+1!=5 - ** UNION ALL - ** SELECT y+1 FROM tab WHERE y+1!=5 - ** UNION ALL - ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5 - ** ORDER BY 1 - ** - ** We call this the "compound-subquery flattening". - */ - for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){ - Select *pNew; - ExprList *pOrderBy = p->pOrderBy; - Expr *pLimit = p->pLimit; - Select *pPrior = p->pPrior; - p->pOrderBy = 0; - p->pSrc = 0; - p->pPrior = 0; - p->pLimit = 0; - pNew = sqlite3SelectDup(db, p, 0); - p->pLimit = pLimit; - p->pOrderBy = pOrderBy; - p->pSrc = pSrc; - p->op = TK_ALL; - p->pRightmost = 0; - if( pNew==0 ){ - pNew = pPrior; + wsdAutoextInit; + if( wsdAutoext.nExt==0 ){ + /* Common case: early out without every having to acquire a mutex */ + return; + } + for(i=0; go; i++){ + char *zErrmsg; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + if( i>=wsdAutoext.nExt ){ + xInit = 0; + go = 0; }else{ - pNew->pPrior = pPrior; - pNew->pRightmost = 0; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + wsdAutoext.aExt[i]; } - p->pPrior = pNew; - if( db->mallocFailed ) return 1; + sqlite3_mutex_leave(mutex); + zErrmsg = 0; + if( xInit && xInit(db, &zErrmsg, &sqlite3Apis) ){ + sqlite3Error(db, SQLITE_ERROR, + "automatic extension loading failed: %s", zErrmsg); + go = 0; + } + sqlite3_free(zErrmsg); } +} - /* Begin flattening the iFrom-th entry of the FROM clause - ** in the outer query. - */ - pSub = pSub1 = pSubitem->pSelect; - - /* Delete the transient table structure associated with the - ** subquery - */ - sqlite3DbFree(db, pSubitem->zDatabase); - sqlite3DbFree(db, pSubitem->zName); - sqlite3DbFree(db, pSubitem->zAlias); - pSubitem->zDatabase = 0; - pSubitem->zName = 0; - pSubitem->zAlias = 0; - pSubitem->pSelect = 0; +/************** End of loadext.c *********************************************/ +/************** Begin file pragma.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the PRAGMA command. +*/ - /* Defer deleting the Table object associated with the - ** subquery until code generation is - ** complete, since there may still exist Expr.pTab entries that - ** refer to the subquery even after flattening. Ticket #3346. - ** - ** pSubitem->pTab is always non-NULL by test restrictions and tests above. - */ - if( ALWAYS(pSubitem->pTab!=0) ){ - Table *pTabToDel = pSubitem->pTab; - if( pTabToDel->nRef==1 ){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - pTabToDel->pNextZombie = pToplevel->pZombieTab; - pToplevel->pZombieTab = pTabToDel; - }else{ - pTabToDel->nRef--; +/* +** Interpret the given string as a safety level. Return 0 for OFF, +** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or +** unrecognized string argument. +** +** Note that the values returned are one less that the values that +** should be passed into sqlite3BtreeSetSafetyLevel(). The is done +** to support legacy SQL code. The safety level used to be boolean +** and older scripts may have used numbers 0 for OFF and 1 for ON. +*/ +static u8 getSafetyLevel(const char *z){ + /* 123456789 123456789 */ + static const char zText[] = "onoffalseyestruefull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; + int i, n; + if( sqlite3Isdigit(*z) ){ + return (u8)sqlite3Atoi(z); + } + n = sqlite3Strlen30(z); + for(i=0; ipTab = 0; } + return 1; +} - /* The following loop runs once for each term in a compound-subquery - ** flattening (as described above). If we are doing a different kind - ** of flattening - a flattening other than a compound-subquery flattening - - ** then this loop only runs once. - ** - ** This loop moves all of the FROM elements of the subquery into the - ** the FROM clause of the outer query. Before doing this, remember - ** the cursor number for the original outer query FROM element in - ** iParent. The iParent cursor will never be used. Subsequent code - ** will scan expressions looking for iParent references and replace - ** those references with expressions that resolve to the subquery FROM - ** elements we are now copying in. - */ - for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ - int nSubSrc; - u8 jointype = 0; - pSubSrc = pSub->pSrc; /* FROM clause of subquery */ - nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ - pSrc = pParent->pSrc; /* FROM clause of the outer query */ +/* +** Interpret the given string as a boolean value. +*/ +SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z){ + return getSafetyLevel(z)&1; +} - if( pSrc ){ - assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->jointype; - }else{ - assert( pParent!=p ); /* 2nd and subsequent times through the loop */ - pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); - if( pSrc==0 ){ - assert( db->mallocFailed ); - break; - } - } +/* The sqlite3GetBoolean() function is used by other modules but the +** remainder of this file is specific to PRAGMA processing. So omit +** the rest of the file if PRAGMAs are omitted from the build. +*/ +#if !defined(SQLITE_OMIT_PRAGMA) - /* The subquery uses a single slot of the FROM clause of the outer - ** query. If the subquery has more than one element in its FROM clause, - ** then expand the outer query to make space for it to hold all elements - ** of the subquery. - ** - ** Example: - ** - ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; - ** - ** The outer query has 3 slots in its FROM clause. One slot of the - ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. - */ - if( nSubSrc>1 ){ - pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); - if( db->mallocFailed ){ - break; - } - } - - /* Transfer the FROM clause terms from the subquery into the - ** outer query. - */ - for(i=0; ia[i+iFrom].pUsing); - pSrc->a[i+iFrom] = pSubSrc->a[i]; - memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); - } - pSrc->a[iFrom].jointype = jointype; - - /* Now begin substituting subquery result set expressions for - ** references to the iParent in the outer query. - ** - ** Example: - ** - ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; - ** \ \_____________ subquery __________/ / - ** \_____________________ outer query ______________________________/ - ** - ** We look at every expression in the outer query and every place we see - ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". - */ - pList = pParent->pEList; - for(i=0; inExpr; i++){ - if( pList->a[i].zName==0 ){ - const char *zSpan = pList->a[i].zSpan; - if( ALWAYS(zSpan) ){ - pList->a[i].zName = sqlite3DbStrDup(db, zSpan); - } - } - } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } - if( pSub->pOrderBy ){ - assert( pParent->pOrderBy==0 ); - pParent->pOrderBy = pSub->pOrderBy; - pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; - } - if( subqueryIsAgg ){ - assert( pParent->pHaving==0 ); - pParent->pHaving = pParent->pWhere; - pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, - sqlite3ExprDup(db, pSub->pHaving, 0)); - assert( pParent->pGroupBy==0 ); - pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); - }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); - pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); - } - - /* The flattened query is distinct if either the inner or the - ** outer query is distinct. - */ - pParent->selFlags |= pSub->selFlags & SF_Distinct; - - /* - ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; - ** - ** One is tempted to try to add a and b to combine the limits. But this - ** does not work if either limit is negative. - */ - if( pSub->pLimit ){ - pParent->pLimit = pSub->pLimit; - pSub->pLimit = 0; - } - } - - /* Finially, delete what is left of the subquery and return - ** success. - */ - sqlite3SelectDelete(db, pSub1); - - return 1; -} -#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Interpret the given string as a locking mode value. +*/ +static int getLockingMode(const char *z){ + if( z ){ + if( 0==sqlite3StrICmp(z, "exclusive") ) return PAGER_LOCKINGMODE_EXCLUSIVE; + if( 0==sqlite3StrICmp(z, "normal") ) return PAGER_LOCKINGMODE_NORMAL; + } + return PAGER_LOCKINGMODE_QUERY; +} +#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Analyze the SELECT statement passed as an argument to see if it -** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if -** it is, or 0 otherwise. At present, a query is considered to be -** a min()/max() query if: -** -** 1. There is a single object in the FROM clause. +** Interpret the given string as an auto-vacuum mode value. ** -** 2. There is a single expression in the result set, and it is -** either min(x) or max(x), where x is a column reference. +** The following strings, "none", "full" and "incremental" are +** acceptable, as are their numeric equivalents: 0, 1 and 2 respectively. */ -static u8 minMaxQuery(Select *p){ - Expr *pExpr; - ExprList *pEList = p->pEList; - - if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; - pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; - pEList = pExpr->x.pList; - if( pEList==0 || pEList->nExpr!=1 ) return 0; - if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ - return WHERE_ORDERBY_MIN; - }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ - return WHERE_ORDERBY_MAX; - } - return WHERE_ORDERBY_NORMAL; +static int getAutoVacuum(const char *z){ + int i; + if( 0==sqlite3StrICmp(z, "none") ) return BTREE_AUTOVACUUM_NONE; + if( 0==sqlite3StrICmp(z, "full") ) return BTREE_AUTOVACUUM_FULL; + if( 0==sqlite3StrICmp(z, "incremental") ) return BTREE_AUTOVACUUM_INCR; + i = sqlite3Atoi(z); + return (u8)((i>=0&&i<=2)?i:0); } +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS /* -** The select statement passed as the first argument is an aggregate query. -** The second argment is the associated aggregate-info object. This -** function tests if the SELECT is of the form: -** -** SELECT count(*) FROM -** -** where table is a database table, not a sub-select or view. If the query -** does match this pattern, then a pointer to the Table object representing -** is returned. Otherwise, 0 is returned. +** Interpret the given string as a temp db location. Return 1 for file +** backed temporary databases, 2 for the Red-Black tree in memory database +** and 0 to use the compile-time default. */ -static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ - Table *pTab; - Expr *pExpr; - - assert( !p->pGroupBy ); - - if( p->pWhere || p->pEList->nExpr!=1 - || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect - ){ +static int getTempStore(const char *z){ + if( z[0]>='0' && z[0]<='2' ){ + return z[0] - '0'; + }else if( sqlite3StrICmp(z, "file")==0 ){ + return 1; + }else if( sqlite3StrICmp(z, "memory")==0 ){ + return 2; + }else{ return 0; } - pTab = p->pSrc->a[0].pTab; - pExpr = p->pEList->a[0].pExpr; - assert( pTab && !pTab->pSelect && pExpr ); - - if( IsVirtual(pTab) ) return 0; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( (pAggInfo->aFunc[0].pFunc->flags&SQLITE_FUNC_COUNT)==0 ) return 0; - if( pExpr->flags&EP_Distinct ) return 0; - - return pTab; } +#endif /* SQLITE_PAGER_PRAGMAS */ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS /* -** If the source-list item passed as an argument was augmented with an -** INDEXED BY clause, then try to locate the specified index. If there -** was such a clause and the named index cannot be found, return -** SQLITE_ERROR and leave an error in pParse. Otherwise, populate -** pFrom->pIndex and return SQLITE_OK. +** Invalidate temp storage, either when the temp storage is changed +** from default, or when 'file' and the temp_store_directory has changed */ -SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->zIndex ){ - Table *pTab = pFrom->pTab; - char *zIndex = pFrom->zIndex; - Index *pIdx; - for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndex); - pIdx=pIdx->pNext - ); - if( !pIdx ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); - pParse->checkSchema = 1; +static int invalidateTempStorage(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt!=0 ){ + if( !db->autoCommit || sqlite3BtreeIsInReadTrans(db->aDb[1].pBt) ){ + sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " + "from within a transaction"); return SQLITE_ERROR; } - pFrom->pIndex = pIdx; + sqlite3BtreeClose(db->aDb[1].pBt); + db->aDb[1].pBt = 0; + sqlite3ResetInternalSchema(db, -1); } return SQLITE_OK; } +#endif /* SQLITE_PAGER_PRAGMAS */ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS /* -** This routine is a Walker callback for "expanding" a SELECT statement. -** "Expanding" means to do the following: -** -** (1) Make sure VDBE cursor numbers have been assigned to every -** element of the FROM clause. -** -** (2) Fill in the pTabList->a[].pTab fields in the SrcList that -** defines FROM clause. When views appear in the FROM clause, -** fill pTabList->a[].pSelect with a copy of the SELECT statement -** that implements the view. A copy is made of the view's SELECT -** statement so that we can freely modify or delete that statement -** without worrying about messing up the presistent representation -** of the view. -** -** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword -** on joins and the ON and USING clause of joins. -** -** (4) Scan the list of columns in the result set (pEList) looking -** for instances of the "*" operator or the TABLE.* operator. -** If found, expand each "*" to be every column in every table -** and TABLE.* to be every column in TABLE. -** +** If the TEMP database is open, close it and mark the database schema +** as needing reloading. This must be done when using the SQLITE_TEMP_STORE +** or DEFAULT_TEMP_STORE pragmas. */ -static int selectExpander(Walker *pWalker, Select *p){ - Parse *pParse = pWalker->pParse; - int i, j, k; - SrcList *pTabList; - ExprList *pEList; - struct SrcList_item *pFrom; +static int changeTempStorage(Parse *pParse, const char *zStorageType){ + int ts = getTempStore(zStorageType); sqlite3 *db = pParse->db; - - if( db->mallocFailed ){ - return WRC_Abort; - } - if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ - return WRC_Prune; + if( db->temp_store==ts ) return SQLITE_OK; + if( invalidateTempStorage( pParse ) != SQLITE_OK ){ + return SQLITE_ERROR; } - p->selFlags |= SF_Expanded; - pTabList = p->pSrc; - pEList = p->pEList; + db->temp_store = (u8)ts; + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ - /* Make sure cursor numbers have been assigned to all entries in - ** the FROM clause of the SELECT statement. - */ - sqlite3SrcListAssignCursors(pParse, pTabList); +/* +** Generate code to return a single integer value. +*/ +static void returnSingleInt(Parse *pParse, const char *zLabel, i64 value){ + Vdbe *v = sqlite3GetVdbe(pParse); + int mem = ++pParse->nMem; + i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); + if( pI64 ){ + memcpy(pI64, &value, sizeof(value)); + } + sqlite3VdbeAddOp4(v, OP_Int64, 0, mem, 0, (char*)pI64, P4_INT64); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); +} - /* Look up every table named in the FROM clause of the select. If - ** an entry of the FROM clause is a subquery instead of a table or view, - ** then create a transient table structure to describe the subquery. - */ - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); - return WRC_Prune; - } - if( pFrom->zName==0 ){ -#ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->pSelect; - /* A sub-query in the FROM clause of a SELECT */ - assert( pSel!=0 ); - assert( pFrom->pTab==0 ); - sqlite3WalkSelect(pWalker, pSel); - pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTab==0 ) return WRC_Abort; - pTab->nRef = 1; - pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); - while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); - pTab->iPKey = -1; - pTab->nRowEst = 1000000; - pTab->tabFlags |= TF_Ephemeral; +#ifndef SQLITE_OMIT_FLAG_PRAGMAS +/* +** Check to see if zRight and zLeft refer to a pragma that queries +** or changes one of the flags in db->flags. Return 1 if so and 0 if not. +** Also, implement the pragma. +*/ +static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ + static const struct sPragmaType { + const char *zName; /* Name of the pragma */ + int mask; /* Mask for the db->flags value */ + } aPragma[] = { + { "full_column_names", SQLITE_FullColNames }, + { "short_column_names", SQLITE_ShortColNames }, + { "count_changes", SQLITE_CountRows }, + { "empty_result_callbacks", SQLITE_NullCallback }, + { "legacy_file_format", SQLITE_LegacyFileFmt }, + { "fullfsync", SQLITE_FullFSync }, + { "checkpoint_fullfsync", SQLITE_CkptFullFSync }, + { "reverse_unordered_selects", SQLITE_ReverseOrder }, +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + { "automatic_index", SQLITE_AutoIndex }, #endif - }else{ - /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); - if( pTab==0 ) return WRC_Abort; - pTab->nRef++; -#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) - if( pTab->pSelect || IsVirtual(pTab) ){ - /* We reach here if the named table is a really a view */ - if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->pSelect==0 ); - pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); - sqlite3WalkSelect(pWalker, pFrom->pSelect); - } +#ifdef SQLITE_DEBUG + { "sql_trace", SQLITE_SqlTrace }, + { "vdbe_listing", SQLITE_VdbeListing }, + { "vdbe_trace", SQLITE_VdbeTrace }, #endif - } - - /* Locate the index named by the INDEXED BY clause, if any. */ - if( sqlite3IndexedByLookup(pParse, pFrom) ){ - return WRC_Abort; - } - } - - /* Process NATURAL keywords, and ON and USING clauses of joins. - */ - if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){ - return WRC_Abort; - } +#ifndef SQLITE_OMIT_CHECK + { "ignore_check_constraints", SQLITE_IgnoreChecks }, +#endif + /* The following is VERY experimental */ + { "writable_schema", SQLITE_WriteSchema|SQLITE_RecoveryMode }, + { "omit_readlock", SQLITE_NoReadlock }, - /* For every "*" that occurs in the column list, insert the names of - ** all columns in all tables. And for every TABLE.* insert the names - ** of all columns in TABLE. The parser inserted a special expression - ** with the TK_ALL operator for each "*" that it found in the column list. - ** The following code just has to locate the TK_ALL expressions and expand - ** each one to the list of all columns in all tables. - ** - ** The first loop just checks to see if there are any "*" operators - ** that need expanding. - */ - for(k=0; knExpr; k++){ - Expr *pE = pEList->a[k].pExpr; - if( pE->op==TK_ALL ) break; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); - if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break; - } - if( knExpr ){ - /* - ** If we get here it means the result set contains one or more "*" - ** operators that need to be expanded. Loop through each expression - ** in the result set and expand them one by one. - */ - struct ExprList_item *a = pEList->a; - ExprList *pNew = 0; - int flags = pParse->db->flags; - int longNames = (flags & SQLITE_FullColNames)!=0 - && (flags & SQLITE_ShortColNames)==0; + /* TODO: Maybe it shouldn't be possible to change the ReadUncommitted + ** flag if there are any active statements. */ + { "read_uncommitted", SQLITE_ReadUncommitted }, + { "recursive_triggers", SQLITE_RecTriggers }, - for(k=0; knExpr; k++){ - Expr *pE = a[k].pExpr; - assert( pE->op!=TK_DOT || pE->pRight!=0 ); - if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ - /* This particular expression does not need to be expanded. - */ - pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); - if( pNew ){ - pNew->a[pNew->nExpr-1].zName = a[k].zName; - pNew->a[pNew->nExpr-1].zSpan = a[k].zSpan; - a[k].zName = 0; - a[k].zSpan = 0; - } - a[k].pExpr = 0; - }else{ - /* This expression is a "*" or a "TABLE.*" and needs to be - ** expanded. */ - int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ - if( pE->op==TK_DOT ){ - assert( pE->pLeft!=0 ); - assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); - zTName = pE->pLeft->u.zToken; + /* This flag may only be set if both foreign-key and trigger support + ** are present in the build. */ +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + { "foreign_keys", SQLITE_ForeignKeys }, +#endif + }; + int i; + const struct sPragmaType *p; + for(i=0, p=aPragma; izName)==0 ){ + sqlite3 *db = pParse->db; + Vdbe *v; + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); /* Already allocated by sqlite3Pragma() */ + if( ALWAYS(v) ){ + if( zRight==0 ){ + returnSingleInt(pParse, p->zName, (db->flags & p->mask)!=0 ); }else{ - zTName = 0; - } - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - char *zTabName = pFrom->zAlias; - if( zTabName==0 ){ - zTabName = pTab->zName; - } - if( db->mallocFailed ) break; - if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ - continue; + int mask = p->mask; /* Mask of bits to set or clear. */ + if( db->autoCommit==0 ){ + /* Foreign key support may not be enabled or disabled while not + ** in auto-commit mode. */ + mask &= ~(SQLITE_ForeignKeys); } - tableSeen = 1; - for(j=0; jnCol; j++){ - Expr *pExpr, *pRight; - char *zName = pTab->aCol[j].zName; - char *zColname; /* The computed column name */ - char *zToFree; /* Malloced string that needs to be freed */ - Token sColname; /* Computed column name as a token */ - - /* If a column is marked as 'hidden' (currently only possible - ** for virtual tables), do not include it in the expanded - ** result-set list. - */ - if( IsHiddenColumn(&pTab->aCol[j]) ){ - assert(IsVirtual(pTab)); - continue; - } - if( i>0 && zTName==0 ){ - if( (pFrom->jointype & JT_NATURAL)!=0 - && tableAndColumnIndex(pTabList, i, zName, 0, 0) - ){ - /* In a NATURAL join, omit the join columns from the - ** table to the right of the join */ - continue; - } - if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ - /* In a join with a USING clause, omit columns in the - ** using clause from the table on the right. */ - continue; - } - } - pRight = sqlite3Expr(db, TK_ID, zName); - zColname = zName; - zToFree = 0; - if( longNames || pTabList->nSrc>1 ){ - Expr *pLeft; - pLeft = sqlite3Expr(db, TK_ID, zTabName); - pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - if( longNames ){ - zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); - zToFree = zColname; - } - }else{ - pExpr = pRight; - } - pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sColname.z = zColname; - sColname.n = sqlite3Strlen30(zColname); - sqlite3ExprListSetName(pParse, pNew, &sColname, 0); - sqlite3DbFree(db, zToFree); - } - } - if( !tableSeen ){ - if( zTName ){ - sqlite3ErrorMsg(pParse, "no such table: %s", zTName); + if( sqlite3GetBoolean(zRight) ){ + db->flags |= mask; }else{ - sqlite3ErrorMsg(pParse, "no tables specified"); + db->flags &= ~mask; } + + /* Many of the flag-pragmas modify the code generated by the SQL + ** compiler (eg. count_changes). So add an opcode to expire all + ** compiled SQL statements after modifying a pragma value. + */ + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); } } + + return 1; } - sqlite3ExprListDelete(db, pEList); - p->pEList = pNew; - } -#if SQLITE_MAX_COLUMN - if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ - sqlite3ErrorMsg(pParse, "too many columns in result set"); } -#endif - return WRC_Continue; + return 0; } +#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ /* -** No-op routine for the parse-tree walker. -** -** When this routine is the Walker.xExprCallback then expression trees -** are walked without any actions being taken at each node. Presumably, -** when this routine is used for Walker.xExprCallback then -** Walker.xSelectCallback is set to do something useful for every -** subquery in the parser tree. -*/ -static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - return WRC_Continue; -} - -/* -** This routine "expands" a SELECT statement and all of its subqueries. -** For additional information on what it means to "expand" a SELECT -** statement, see the comment on the selectExpand worker callback above. -** -** Expanding a SELECT statement is the first step in processing a -** SELECT statement. The SELECT statement must be expanded before -** name resolution is performed. -** -** If anything goes wrong, an error message is written into pParse. -** The calling function can detect the problem by looking at pParse->nErr -** and/or pParse->db->mallocFailed. -*/ -static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ - Walker w; - w.xSelectCallback = selectExpander; - w.xExprCallback = exprWalkNoop; - w.pParse = pParse; - sqlite3WalkSelect(&w, pSelect); -} - - -#ifndef SQLITE_OMIT_SUBQUERY -/* -** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() -** interface. -** -** For each FROM-clause subquery, add Column.zType and Column.zColl -** information to the Table structure that represents the result set -** of that subquery. -** -** The Table structure that represents the result set was constructed -** by selectExpander() but the type and collation information was omitted -** at that point because identifiers had not yet been resolved. This -** routine is called after identifier resolution. +** Return a human-readable name for a constraint resolution action. */ -static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ - Parse *pParse; - int i; - SrcList *pTabList; - struct SrcList_item *pFrom; - - assert( p->selFlags & SF_Resolved ); - if( (p->selFlags & SF_HasTypeInfo)==0 ){ - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - assert( pSel ); - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); - } - } +#ifndef SQLITE_OMIT_FOREIGN_KEY +static const char *actionName(u8 action){ + const char *zName; + switch( action ){ + case OE_SetNull: zName = "SET NULL"; break; + case OE_SetDflt: zName = "SET DEFAULT"; break; + case OE_Cascade: zName = "CASCADE"; break; + case OE_Restrict: zName = "RESTRICT"; break; + default: zName = "NO ACTION"; + assert( action==OE_None ); break; } - return WRC_Continue; + return zName; } #endif /* -** This routine adds datatype and collating sequence information to -** the Table structures of all FROM-clause subqueries in a -** SELECT statement. -** -** Use this routine after name resolution. +** Parameter eMode must be one of the PAGER_JOURNALMODE_XXX constants +** defined in pager.h. This function returns the associated lowercase +** journal-mode name. */ -static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ -#ifndef SQLITE_OMIT_SUBQUERY - Walker w; - w.xSelectCallback = selectAddSubqueryTypeInfo; - w.xExprCallback = exprWalkNoop; - w.pParse = pParse; - sqlite3WalkSelect(&w, pSelect); +SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ + static char * const azModeName[] = { + "delete", "persist", "off", "truncate", "memory" +#ifndef SQLITE_OMIT_WAL + , "wal" #endif -} + }; + assert( PAGER_JOURNALMODE_DELETE==0 ); + assert( PAGER_JOURNALMODE_PERSIST==1 ); + assert( PAGER_JOURNALMODE_OFF==2 ); + assert( PAGER_JOURNALMODE_TRUNCATE==3 ); + assert( PAGER_JOURNALMODE_MEMORY==4 ); + assert( PAGER_JOURNALMODE_WAL==5 ); + assert( eMode>=0 && eMode<=ArraySize(azModeName) ); + if( eMode==ArraySize(azModeName) ) return 0; + return azModeName[eMode]; +} /* -** This routine sets of a SELECT statement for processing. The -** following is accomplished: +** Process a pragma statement. ** -** * VDBE Cursor numbers are assigned to all FROM-clause terms. -** * Ephemeral Table objects are created for all FROM-clause subqueries. -** * ON and USING clauses are shifted into WHERE statements -** * Wildcards "*" and "TABLE.*" in result sets are expanded. -** * Identifiers in expression are matched to tables. +** Pragmas are of this form: ** -** This routine acts recursively on all subqueries within the SELECT. +** PRAGMA [database.]id [= value] +** +** The identifier might also be a string. The value is a string, and +** identifier, or a number. If minusFlag is true, then the value is +** a number that was preceded by a minus sign. +** +** If the left side is "database.id" then pId1 is the database name +** and pId2 is the id. If the left side is just "id" then pId1 is the +** id and pId2 is any empty string. */ -SQLITE_PRIVATE void sqlite3SelectPrep( - Parse *pParse, /* The parser context */ - Select *p, /* The SELECT statement being coded. */ - NameContext *pOuterNC /* Name context for container */ +SQLITE_PRIVATE void sqlite3Pragma( + Parse *pParse, + Token *pId1, /* First part of [database.]id field */ + Token *pId2, /* Second part of [database.]id field, or NULL */ + Token *pValue, /* Token for , or NULL */ + int minusFlag /* True if a '-' sign preceded */ ){ - sqlite3 *db; - if( NEVER(p==0) ) return; - db = pParse->db; - if( p->selFlags & SF_HasTypeInfo ) return; - sqlite3SelectExpand(pParse, p); - if( pParse->nErr || db->mallocFailed ) return; - sqlite3ResolveSelectNames(pParse, p, pOuterNC); - if( pParse->nErr || db->mallocFailed ) return; - sqlite3SelectAddTypeInfo(pParse, p); -} + char *zLeft = 0; /* Nul-terminated UTF-8 string */ + char *zRight = 0; /* Nul-terminated UTF-8 string , or NULL */ + const char *zDb = 0; /* The database name */ + Token *pId; /* Pointer to token */ + int iDb; /* Database index for */ + sqlite3 *db = pParse->db; + Db *pDb; + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); + if( v==0 ) return; + sqlite3VdbeRunOnlyOnce(v); + pParse->nMem = 2; -/* -** Reset the aggregate accumulator. -** -** The aggregate accumulator is a set of memory cells that hold -** intermediate results while calculating an aggregate. This -** routine simply stores NULLs in all of those memory cells. -*/ -static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ - Vdbe *v = pParse->pVdbe; - int i; - struct AggInfo_func *pFunc; - if( pAggInfo->nFunc+pAggInfo->nColumn==0 ){ + /* Interpret the [database.] part of the pragma statement. iDb is the + ** index of the database this pragma is being applied to in db.aDb[]. */ + iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); + if( iDb<0 ) return; + pDb = &db->aDb[iDb]; + + /* If the temp database has been explicitly named as part of the + ** pragma, make sure it is open. + */ + if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ return; } - for(i=0; inColumn; i++){ - sqlite3VdbeAddOp2(v, OP_Null, 0, pAggInfo->aCol[i].iMem); + + zLeft = sqlite3NameFromToken(db, pId); + if( !zLeft ) return; + if( minusFlag ){ + zRight = sqlite3MPrintf(db, "-%T", pValue); + }else{ + zRight = sqlite3NameFromToken(db, pValue); } - for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ - sqlite3VdbeAddOp2(v, OP_Null, 0, pFunc->iMem); - if( pFunc->iDistinct>=0 ){ - Expr *pE = pFunc->pExpr; - assert( !ExprHasProperty(pE, EP_xIsSelect) ); - if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ - sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " - "argument"); - pFunc->iDistinct = -1; - }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + + assert( pId2 ); + zDb = pId2->n>0 ? pDb->zName : 0; + if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ + goto pragma_out; + } + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + /* + ** PRAGMA [database.]default_cache_size + ** PRAGMA [database.]default_cache_size=N + ** + ** The first form reports the current persistent setting for the + ** page cache size. The value returned is the maximum number of + ** pages in the page cache. The second form sets both the current + ** page cache size value and the persistent page cache size value + ** stored in the database file. + ** + ** Older versions of SQLite would set the default cache size to a + ** negative number to indicate synchronous=OFF. These days, synchronous + ** is always on by default regardless of the sign of the default cache + ** size. But continue to take the absolute value of the default cache + ** size of historical compatibility. + */ + if( sqlite3StrICmp(zLeft,"default_cache_size")==0 ){ + static const VdbeOpList getCacheSize[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ + { OP_IfPos, 1, 7, 0}, + { OP_Integer, 0, 2, 0}, + { OP_Subtract, 1, 2, 1}, + { OP_IfPos, 1, 7, 0}, + { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_ResultRow, 1, 1, 0}, + }; + int addr; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeUsesBtree(v, iDb); + if( !zRight ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + pParse->nMem += 2; + addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, iDb); + sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); + }else{ + int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, size, 1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + }else + + /* + ** PRAGMA [database.]page_size + ** PRAGMA [database.]page_size=N + ** + ** The first form reports the current setting for the + ** database page size in bytes. The second form sets the + ** database page size value. The value can only be set if + ** the database has not yet been created. + */ + if( sqlite3StrICmp(zLeft,"page_size")==0 ){ + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; + returnSingleInt(pParse, "page_size", size); + }else{ + /* Malloc may fail when setting the page-size, as there is an internal + ** buffer that the pager module resizes using sqlite3_realloc(). + */ + db->nextPagesize = sqlite3Atoi(zRight); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + db->mallocFailed = 1; } } - } -} + }else -/* -** Invoke the OP_AggFinalize opcode for every aggregate function -** in the AggInfo structure. -*/ -static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ - Vdbe *v = pParse->pVdbe; - int i; - struct AggInfo_func *pF; - for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ - ExprList *pList = pF->pExpr->x.pList; - assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); - sqlite3VdbeAddOp4(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, 0, - (void*)pF->pFunc, P4_FUNCDEF); - } -} + /* + ** PRAGMA [database.]secure_delete + ** PRAGMA [database.]secure_delete=ON/OFF + ** + ** The first form reports the current setting for the + ** secure_delete flag. The second form changes the secure_delete + ** flag setting and reports thenew value. + */ + if( sqlite3StrICmp(zLeft,"secure_delete")==0 ){ + Btree *pBt = pDb->pBt; + int b = -1; + assert( pBt!=0 ); + if( zRight ){ + b = sqlite3GetBoolean(zRight); + } + if( pId2->n==0 && b>=0 ){ + int ii; + for(ii=0; iinDb; ii++){ + sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); + } + } + b = sqlite3BtreeSecureDelete(pBt, b); + returnSingleInt(pParse, "secure_delete", b); + }else -/* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. -*/ -static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ - Vdbe *v = pParse->pVdbe; - int i; - struct AggInfo_func *pF; - struct AggInfo_col *pC; + /* + ** PRAGMA [database.]max_page_count + ** PRAGMA [database.]max_page_count=N + ** + ** The first form reports the current setting for the + ** maximum number of pages in the database file. The + ** second form attempts to change this setting. Both + ** forms return the current setting. + ** + ** PRAGMA [database.]page_count + ** + ** Return the number of pages in the specified database. + */ + if( sqlite3StrICmp(zLeft,"page_count")==0 + || sqlite3StrICmp(zLeft,"max_page_count")==0 + ){ + int iReg; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3CodeVerifySchema(pParse, iDb); + iReg = ++pParse->nMem; + if( zLeft[0]=='p' ){ + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + }else{ + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + }else - pAggInfo->directMode = 1; - sqlite3ExprCacheClear(pParse); - for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ - int nArg; - int addrNext = 0; - int regAgg; - ExprList *pList = pF->pExpr->x.pList; - assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); - if( pList ){ - nArg = pList->nExpr; - regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, 1); + /* + ** PRAGMA [database.]locking_mode + ** PRAGMA [database.]locking_mode = (normal|exclusive) + */ + if( sqlite3StrICmp(zLeft,"locking_mode")==0 ){ + const char *zRet = "normal"; + int eMode = getLockingMode(zRight); + + if( pId2->n==0 && eMode==PAGER_LOCKINGMODE_QUERY ){ + /* Simple "PRAGMA locking_mode;" statement. This is a query for + ** the current default locking mode (which may be different to + ** the locking-mode of the main database). + */ + eMode = db->dfltLockMode; }else{ - nArg = 0; - regAgg = 0; + Pager *pPager; + if( pId2->n==0 ){ + /* This indicates that no database name was specified as part + ** of the PRAGMA command. In this case the locking-mode must be + ** set on all attached databases, as well as the main db file. + ** + ** Also, the sqlite3.dfltLockMode variable is set so that + ** any subsequently attached databases also use the specified + ** locking mode. + */ + int ii; + assert(pDb==&db->aDb[0]); + for(ii=2; iinDb; ii++){ + pPager = sqlite3BtreePager(db->aDb[ii].pBt); + sqlite3PagerLockingMode(pPager, eMode); + } + db->dfltLockMode = (u8)eMode; + } + pPager = sqlite3BtreePager(pDb->pBt); + eMode = sqlite3PagerLockingMode(pPager, eMode); } - if( pF->iDistinct>=0 ){ - addrNext = sqlite3VdbeMakeLabel(v); - assert( nArg==1 ); - codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); + + assert(eMode==PAGER_LOCKINGMODE_NORMAL||eMode==PAGER_LOCKINGMODE_EXCLUSIVE); + if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ + zRet = "exclusive"; } - if( pF->pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ - CollSeq *pColl = 0; - struct ExprList_item *pItem; - int j; - assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ - for(j=0, pItem=pList->a; !pColl && jpExpr); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + }else + + /* + ** PRAGMA [database.]journal_mode + ** PRAGMA [database.]journal_mode = + ** (delete|persist|off|truncate|memory|wal|off) + */ + if( sqlite3StrICmp(zLeft,"journal_mode")==0 ){ + int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ + int ii; /* Loop counter */ + + /* Force the schema to be loaded on all databases. This cases all + ** database files to be opened and the journal_modes set. */ + if( sqlite3ReadSchema(pParse) ){ + goto pragma_out; + } + + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); + + if( zRight==0 ){ + /* If there is no "=MODE" part of the pragma, do a query for the + ** current mode */ + eMode = PAGER_JOURNALMODE_QUERY; + }else{ + const char *zMode; + int n = sqlite3Strlen30(zRight); + for(eMode=0; (zMode = sqlite3JournalModename(eMode))!=0; eMode++){ + if( sqlite3StrNICmp(zRight, zMode, n)==0 ) break; } - if( !pColl ){ - pColl = pParse->db->pDfltColl; + if( !zMode ){ + /* If the "=MODE" part does not match any known journal mode, + ** then do a query */ + eMode = PAGER_JOURNALMODE_QUERY; } - sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, - (void*)pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); - sqlite3ReleaseTempRange(pParse, regAgg, nArg); - if( addrNext ){ - sqlite3VdbeResolveLabel(v, addrNext); - sqlite3ExprCacheClear(pParse); + if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ + /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ + iDb = 0; + pId2->n = 1; } - } + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3VdbeUsesBtree(v, ii); + sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode); + } + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + }else - /* Before populating the accumulator registers, clear the column cache. - ** Otherwise, if any of the required column values are already present - ** in registers, sqlite3ExprCode() may use OP_SCopy to copy the value - ** to pC->iMem. But by the time the value is used, the original register - ** may have been used, invalidating the underlying buffer holding the - ** text or blob value. See ticket [883034dcb5]. + /* + ** PRAGMA [database.]journal_size_limit + ** PRAGMA [database.]journal_size_limit=N ** - ** Another solution would be to change the OP_SCopy used to copy cached - ** values to an OP_Copy. + ** Get or set the size limit on rollback journal files. */ - sqlite3ExprCacheClear(pParse); - for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); - } - pAggInfo->directMode = 0; - sqlite3ExprCacheClear(pParse); -} + if( sqlite3StrICmp(zLeft,"journal_size_limit")==0 ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + i64 iLimit = -2; + if( zRight ){ + sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); + if( iLimit<-1 ) iLimit = -1; + } + iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); + returnSingleInt(pParse, "journal_size_limit", iLimit); + }else -/* -** Generate code for the SELECT statement given in the p argument. -** -** The results are distributed in various ways depending on the -** contents of the SelectDest structure pointed to by argument pDest -** as follows: -** -** pDest->eDest Result -** ------------ ------------------------------------------- -** SRT_Output Generate a row of output (using the OP_ResultRow -** opcode) for each row in the result set. -** -** SRT_Mem Only valid if the result is a single column. -** Store the first column of the first result row -** in register pDest->iParm then abandon the rest -** of the query. This destination implies "LIMIT 1". -** -** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iParm. -** Apply the affinity pDest->affinity before storing -** results. Used to implement "IN (SELECT ...)". -** -** SRT_Union Store results as a key in a temporary table pDest->iParm. -** -** SRT_Except Remove results from the temporary table pDest->iParm. -** -** SRT_Table Store results in temporary table pDest->iParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** -** SRT_EphemTab Create an temporary table pDest->iParm and store -** the result there. The cursor is left open after -** returning. This is like SRT_Table except that -** this destination uses OP_OpenEphemeral to create -** the table first. -** -** SRT_Coroutine Generate a co-routine that returns a new row of -** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iParm. -** -** SRT_Exists Store a 1 in memory cell pDest->iParm if the result -** set is not empty. -** -** SRT_Discard Throw the results away. This is used by SELECT -** statements within triggers whose only purpose is -** the side-effects of functions. -** -** This routine returns the number of errors. If any errors are -** encountered, then an appropriate error message is left in -** pParse->zErrMsg. -** -** This routine does NOT free the Select structure passed in. The -** calling function needs to do that. -*/ -SQLITE_PRIVATE int sqlite3Select( - Parse *pParse, /* The parser context */ - Select *p, /* The SELECT statement being coded. */ - SelectDest *pDest /* What to do with the query results */ -){ - int i, j; /* Loop counters */ - WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ - Vdbe *v; /* The virtual machine under construction */ - int isAgg; /* True for select lists like "count(*)" */ - ExprList *pEList; /* List of columns to extract. */ - SrcList *pTabList; /* List of tables to select from */ - Expr *pWhere; /* The WHERE clause. May be NULL */ - ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ - ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ - Expr *pHaving; /* The HAVING clause. May be NULL */ - int isDistinct; /* True if the DISTINCT keyword is present */ - int distinct; /* Table to use for the distinct set */ - int rc = 1; /* Value to return from this function */ - int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ - AggInfo sAggInfo; /* Information used by aggregate queries */ - int iEnd; /* Address of the end of the query */ - sqlite3 *db; /* The database connection */ - -#ifndef SQLITE_OMIT_EXPLAIN - int iRestoreSelectId = pParse->iSelectId; - pParse->iSelectId = pParse->iNextSelectId++; -#endif - - db = pParse->db; - if( p==0 || db->mallocFailed || pParse->nErr ){ - return 1; - } - if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; - memset(&sAggInfo, 0, sizeof(sAggInfo)); - - if( IgnorableOrderby(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); - /* If ORDER BY makes no difference in the output then neither does - ** DISTINCT so it can be removed too. */ - sqlite3ExprListDelete(db, p->pOrderBy); - p->pOrderBy = 0; - p->selFlags &= ~SF_Distinct; - } - sqlite3SelectPrep(pParse, p, 0); - pOrderBy = p->pOrderBy; - pTabList = p->pSrc; - pEList = p->pEList; - if( pParse->nErr || db->mallocFailed ){ - goto select_end; - } - isAgg = (p->selFlags & SF_Aggregate)!=0; - assert( pEList!=0 ); - - /* Begin generating code. - */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto select_end; - - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - goto select_end; - } -#endif +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ - /* Generate code for all sub-queries in the FROM clause + /* + ** PRAGMA [database.]auto_vacuum + ** PRAGMA [database.]auto_vacuum=N + ** + ** Get or set the value of the database 'auto-vacuum' parameter. + ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL */ -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; !p->pPrior && inSrc; i++){ - struct SrcList_item *pItem = &pTabList->a[i]; - SelectDest dest; - Select *pSub = pItem->pSelect; - int isAggSub; - - if( pSub==0 || pItem->isPopulated ) continue; - - /* Increment Parse.nHeight by the height of the largest expression - ** tree refered to by this, the parent select. The child select - ** may contain expression trees of at most - ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit - ** more conservative than necessary, but much easier than enforcing - ** an exact limit. - */ - pParse->nHeight += sqlite3SelectExprHeight(p); - - /* Check to see if the subquery can be absorbed into the parent. */ - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( sqlite3StrICmp(zLeft,"auto_vacuum")==0 ){ + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( sqlite3ReadSchema(pParse) ){ + goto pragma_out; + } + if( !zRight ){ + int auto_vacuum; + if( ALWAYS(pBt) ){ + auto_vacuum = sqlite3BtreeGetAutoVacuum(pBt); + }else{ + auto_vacuum = SQLITE_DEFAULT_AUTOVACUUM; } - i = -1; + returnSingleInt(pParse, "auto_vacuum", auto_vacuum); }else{ - sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - assert( pItem->isPopulated==0 ); - explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); - sqlite3Select(pParse, pSub, &dest); - pItem->isPopulated = 1; - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; - } - if( /*pParse->nErr ||*/ db->mallocFailed ){ - goto select_end; - } - pParse->nHeight -= sqlite3SelectExprHeight(p); - pTabList = p->pSrc; - if( !IgnorableOrderby(pDest) ){ - pOrderBy = p->pOrderBy; + int eAuto = getAutoVacuum(zRight); + assert( eAuto>=0 && eAuto<=2 ); + db->nextAutovac = (u8)eAuto; + if( ALWAYS(eAuto>=0) ){ + /* Call SetAutoVacuum() to set initialize the internal auto and + ** incr-vacuum flags. This is required in case this connection + ** creates the database file. It is important that it is created + ** as an auto-vacuum capable db. + */ + int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ + /* When setting the auto_vacuum mode to either "full" or + ** "incremental", write the value of meta[6] in the database + ** file. Before writing to meta[6], check that meta[3] indicates + ** that this really is an auto-vacuum capable database. + */ + static const VdbeOpList setMeta6[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, + { OP_If, 1, 0, 0}, /* 2 */ + { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ + { OP_Integer, 0, 1, 0}, /* 4 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ + }; + int iAddr; + iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6); + sqlite3VdbeChangeP1(v, iAddr, iDb); + sqlite3VdbeChangeP1(v, iAddr+1, iDb); + sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); + sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); + sqlite3VdbeChangeP1(v, iAddr+5, iDb); + sqlite3VdbeUsesBtree(v, iDb); + } + } } - } - pEList = p->pEList; + }else #endif - pWhere = p->pWhere; - pGroupBy = p->pGroupBy; - pHaving = p->pHaving; - isDistinct = (p->selFlags & SF_Distinct)!=0; -#ifndef SQLITE_OMIT_COMPOUND_SELECT - /* If there is are a sequence of queries, do the earlier ones first. + /* + ** PRAGMA [database.]incremental_vacuum(N) + ** + ** Do N steps of incremental vacuuming on a database. */ - if( p->pPrior ){ - if( p->pRightmost==0 ){ - Select *pLoop, *pRight = 0; - int cnt = 0; - int mxSelect; - for(pLoop=p; pLoop; pLoop=pLoop->pPrior, cnt++){ - pLoop->pRightmost = p; - pLoop->pNext = pRight; - pRight = pLoop; - } - mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; - if( mxSelect && cnt>mxSelect ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - goto select_end; - } +#ifndef SQLITE_OMIT_AUTOVACUUM + if( sqlite3StrICmp(zLeft,"incremental_vacuum")==0 ){ + int iLimit, addr; + if( sqlite3ReadSchema(pParse) ){ + goto pragma_out; } - rc = multiSelect(pParse, p, pDest); - explainSetInteger(pParse->iSelectId, iRestoreSelectId); - return rc; - } + if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ + iLimit = 0x7fffffff; + } + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); + sqlite3VdbeAddOp1(v, OP_ResultRow, 1); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); + sqlite3VdbeJumpHere(v, addr); + }else #endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. - ** GROUP BY might use an index, DISTINCT never does. - */ - assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; - p->selFlags &= ~SF_Distinct; - } - - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then disable the ORDER BY clause since the GROUP BY - ** will cause elements to come out in the correct order. This is - ** an optimization - the correct answer should result regardless. - ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER - ** to disable this optimization for testing purposes. - */ - if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 - && (db->flags & SQLITE_GroupByOrder)==0 ){ - pOrderBy = 0; - } - - /* If there is an ORDER BY clause, then this sorting - ** index might end up being unused if the data can be - ** extracted in pre-sorted order. If that is the case, then the - ** OP_OpenEphemeral instruction will be changed to an OP_Noop once - ** we figure out that the sorting index is not needed. The addrSortIndex - ** variable is used to facilitate that change. - */ - if( pOrderBy ){ - KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); - pOrderBy->iECursor = pParse->nTab++; - p->addrOpenEphm[2] = addrSortIndex = - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - pOrderBy->iECursor, pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); - }else{ - addrSortIndex = -1; - } - - /* If the output is destined for a temporary table, open that table. +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + /* + ** PRAGMA [database.]cache_size + ** PRAGMA [database.]cache_size=N + ** + ** The first form reports the current local setting for the + ** page cache size. The local setting can be different from + ** the persistent cache size value that is stored in the database + ** file itself. The value returned is the maximum number of + ** pages in the page cache. The second form sets the local + ** page cache size value. It does not change the persistent + ** cache size stored on the disk so the cache size will revert + ** to its default value when the database is closed and reopened. + ** N should be a positive integer. */ - if( pDest->eDest==SRT_EphemTab ){ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iParm, pEList->nExpr); - } + if( sqlite3StrICmp(zLeft,"cache_size")==0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); + }else{ + int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + }else - /* Set the limiter. + /* + ** PRAGMA temp_store + ** PRAGMA temp_store = "default"|"memory"|"file" + ** + ** Return or set the local value of the temp_store flag. Changing + ** the local value does not make changes to the disk file and the default + ** value will be restored the next time the database is opened. + ** + ** Note that it is possible for the library compile-time options to + ** override this setting */ - iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = (double)LARGEST_INT64; - computeLimitRegisters(pParse, p, iEnd); + if( sqlite3StrICmp(zLeft, "temp_store")==0 ){ + if( !zRight ){ + returnSingleInt(pParse, "temp_store", db->temp_store); + }else{ + changeTempStorage(pParse, zRight); + } + }else - /* Open a virtual index to use for the distinct set. + /* + ** PRAGMA temp_store_directory + ** PRAGMA temp_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the temp_store_directory flag. Changing + ** the value sets a specific directory to be used for temporary files. + ** Setting to a null string reverts to the default temporary directory search. + ** If temporary directory is changed, then invalidateTempStorage. + ** */ - if( p->selFlags & SF_Distinct ){ - KeyInfo *pKeyInfo; - assert( isAgg || pGroupBy ); - distinct = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - }else{ - distinct = -1; - } - - /* Aggregate and non-aggregate queries are handled differently */ - if( !isAgg && pGroupBy==0 ){ - /* This case is for non-aggregate queries - ** Begin the database scan - */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); - if( pWInfo==0 ) goto select_end; - if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; - - /* If sorting index that was created by a prior OP_OpenEphemeral - ** instruction ended up not being needed, then change the OP_OpenEphemeral - ** into an OP_Noop. - */ - if( addrSortIndex>=0 && pOrderBy==0 ){ - sqlite3VdbeChangeToNoop(v, addrSortIndex, 1); - p->addrOpenEphm[2] = -1; + if( sqlite3StrICmp(zLeft, "temp_store_directory")==0 ){ + if( !zRight ){ + if( sqlite3_temp_directory ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "temp_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int rc; + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + if( SQLITE_TEMP_STORE==0 + || (SQLITE_TEMP_STORE==1 && db->temp_store<=1) + || (SQLITE_TEMP_STORE==2 && db->temp_store==1) + ){ + invalidateTempStorage(pParse); + } + sqlite3_free(sqlite3_temp_directory); + if( zRight[0] ){ + sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_temp_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ } + }else - /* Use the standard inner loop - */ - assert(!isDistinct); - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, - pWInfo->iContinue, pWInfo->iBreak); - - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); - }else{ - /* This is the processing for aggregate queries */ - NameContext sNC; /* Name context for processing aggregate information */ - int iAMem; /* First Mem address for storing current GROUP BY */ - int iBMem; /* First Mem address for previous GROUP BY */ - int iUseFlag; /* Mem address holding flag indicating that at least - ** one row of the input to the aggregator has been - ** processed */ - int iAbortFlag; /* Mem address which causes query abort if positive */ - int groupBySort; /* Rows come from source in GROUP BY order */ - int addrEnd; /* End of processing for this SELECT */ - - /* Remove any and all aliases between the result set and the - ** GROUP BY clause. - */ - if( pGroupBy ){ - int k; /* Loop counter */ - struct ExprList_item *pItem; /* For looping over expression in a list */ - - for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ - pItem->iAlias = 0; - } - for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ - pItem->iAlias = 0; +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + /* + ** PRAGMA [database.]lock_proxy_file + ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" + ** + ** Return or set the value of the lock_proxy_file flag. Changing + ** the value sets a specific file to be used for database access locks. + ** + */ + if( sqlite3StrICmp(zLeft, "lock_proxy_file")==0 ){ + if( !zRight ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + char *proxy_file_path = NULL; + sqlite3_file *pFile = sqlite3PagerFile(pPager); + sqlite3OsFileControl(pFile, SQLITE_GET_LOCKPROXYFILE, + &proxy_file_path); + + if( proxy_file_path ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "lock_proxy_file", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100; }else{ - p->nSelectRow = (double)1; + Pager *pPager = sqlite3BtreePager(pDb->pBt); + sqlite3_file *pFile = sqlite3PagerFile(pPager); + int res; + if( zRight[0] ){ + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + zRight); + } else { + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + NULL); + } + if( res!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "failed to set lock proxy file"); + goto pragma_out; + } } - - - /* Create a label to jump to when we want to abort the query */ - addrEnd = sqlite3VdbeMakeLabel(v); - - /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in - ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the - ** SELECT statement. - */ - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - sNC.pSrcList = pTabList; - sNC.pAggInfo = &sAggInfo; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; - sAggInfo.pGroupBy = pGroupBy; - sqlite3ExprAnalyzeAggList(&sNC, pEList); - sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); - if( pHaving ){ - sqlite3ExprAnalyzeAggregates(&sNC, pHaving); - } - sAggInfo.nAccumulator = sAggInfo.nColumn; - for(i=0; ix.pList); + }else +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + + /* + ** PRAGMA [database.]synchronous + ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL + ** + ** Return or set the local value of the synchronous flag. Changing + ** the local value does not make changes to the disk file and the + ** default value will be restored the next time the database is + ** opened. + */ + if( sqlite3StrICmp(zLeft,"synchronous")==0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + if( !zRight ){ + returnSingleInt(pParse, "synchronous", pDb->safety_level-1); + }else{ + if( !db->autoCommit ){ + sqlite3ErrorMsg(pParse, + "Safety level may not be changed inside a transaction"); + }else{ + pDb->safety_level = getSafetyLevel(zRight)+1; + } } - if( db->mallocFailed ) goto select_end; + }else +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ - /* Processing for aggregates with GROUP BY is very different and - ** much more complex than aggregates without a GROUP BY. - */ - if( pGroupBy ){ - KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int j1; /* A-vs-B comparision jump */ - int addrOutputRow; /* Start of subroutine that outputs a result row */ - int regOutputRow; /* Return address register for output subroutine */ - int addrSetAbort; /* Set the abort flag and return */ - int addrTopOfLoop; /* Top of the input loop */ - int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ - int addrReset; /* Subroutine for resetting the accumulator */ - int regReset; /* Return address register for reset subroutine */ +#ifndef SQLITE_OMIT_FLAG_PRAGMAS + if( flagPragma(pParse, zLeft, zRight) ){ + /* The flagPragma() subroutine also generates any necessary code + ** there is nothing more to do here */ + }else +#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ - /* If there is a GROUP BY clause we might need a sorting index to - ** implement it. Allocate that sorting index now. If it turns out - ** that we do not need it after all, the OpenEphemeral instruction - ** will be converted into a Noop. - */ - sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sAggInfo.sortingIdx, sAggInfo.nSortingColumn, - 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); +#ifndef SQLITE_OMIT_SCHEMA_PRAGMAS + /* + ** PRAGMA table_info(
    ) + ** + ** Return a single row for each column of the named table. The columns of + ** the returned data set are: + ** + ** cid: Column id (numbered from left to right, starting at 0) + ** name: Column name + ** type: Column declaration type. + ** notnull: True if 'NOT NULL' is part of column declaration + ** dflt_value: The default value for the column, if any. + */ + if( sqlite3StrICmp(zLeft, "table_info")==0 && zRight ){ + Table *pTab; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + int i; + int nHidden = 0; + Column *pCol; + sqlite3VdbeSetNumCols(v, 6); + pParse->nMem = 6; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); + sqlite3ViewGetColumnNames(pParse, pTab); + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + if( IsHiddenColumn(pCol) ){ + nHidden++; + continue; + } + sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + pCol->zType ? pCol->zType : "", 0); + sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); + if( pCol->zDflt ){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, 5); + } + sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); + } + } + }else - /* Initialize memory locations used by GROUP BY aggregate processing - */ - iUseFlag = ++pParse->nMem; - iAbortFlag = ++pParse->nMem; - regOutputRow = ++pParse->nMem; - addrOutputRow = sqlite3VdbeMakeLabel(v); - regReset = ++pParse->nMem; - addrReset = sqlite3VdbeMakeLabel(v); - iAMem = pParse->nMem + 1; - pParse->nMem += pGroupBy->nExpr; - iBMem = pParse->nMem + 1; - pParse->nMem += pGroupBy->nExpr; - sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); - VdbeComment((v, "clear abort flag")); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); + if( sqlite3StrICmp(zLeft, "index_info")==0 && zRight ){ + Index *pIdx; + Table *pTab; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx ){ + int i; + pTab = pIdx->pTable; + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); + for(i=0; inColumn; i++){ + int cnum = pIdx->aiColumn[i]; + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); + assert( pTab->nCol>cnum ); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + } + }else - /* Begin a loop that will extract all source rows in GROUP BY order. - ** This might involve two separate loops with an OP_Sort in between, or - ** it might be a single loop that uses an index to extract information - ** in the right order to begin with. - */ - sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); - if( pWInfo==0 ) goto select_end; - if( pGroupBy==0 ){ - /* The optimizer is able to deliver rows in group by order so - ** we do not have to sort. The OP_OpenEphemeral table will be - ** cancelled later because we still need to use the pKeyInfo - */ - pGroupBy = p->pGroupBy; - groupBySort = 0; - }else{ - /* Rows are coming out in undetermined order. We have to push - ** each row into a sorting index, terminate the first loop, - ** then loop over the sorting index in order to get the output - ** in sorted order - */ - int regBase; - int regRecord; - int nCol; - int nGroupBy; + if( sqlite3StrICmp(zLeft, "index_list")==0 && zRight ){ + Index *pIdx; + Table *pTab; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + v = sqlite3GetVdbe(pParse); + pIdx = pTab->pIndex; + if( pIdx ){ + int i = 0; + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + while(pIdx){ + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + ++i; + pIdx = pIdx->pNext; + } + } + } + }else - explainTempTable(pParse, - isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + if( sqlite3StrICmp(zLeft, "database_list")==0 ){ + int i; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); + for(i=0; inDb; i++){ + if( db->aDb[i].pBt==0 ) continue; + assert( db->aDb[i].zName!=0 ); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + }else - groupBySort = 1; - nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; - for(i=0; i=j ){ - nCol++; - j++; - } - } - regBase = sqlite3GetTempRange(pParse, nCol); - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; - for(i=0; iiSorterColumn>=j ){ - int r1 = j + regBase; - int r2; + if( sqlite3StrICmp(zLeft, "collation_list")==0 ){ + int i = 0; + HashElem *p; + sqlite3VdbeSetNumCols(v, 2); + pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(p); + sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); + } + }else +#endif /* SQLITE_OMIT_SCHEMA_PRAGMAS */ - r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); - if( r1!=r2 ){ - sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); - } - j++; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( sqlite3StrICmp(zLeft, "foreign_key_list")==0 && zRight ){ + FKey *pFK; + Table *pTab; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + v = sqlite3GetVdbe(pParse); + pFK = pTab->pFKey; + if( pFK ){ + int i = 0; + sqlite3VdbeSetNumCols(v, 8); + pParse->nMem = 8; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); + while(pFK){ + int j; + for(j=0; jnCol; j++){ + char *zCol = pFK->aCol[j].zCol; + char *zOnDelete = (char *)actionName(pFK->aAction[0]); + char *zOnUpdate = (char *)actionName(pFK->aAction[1]); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, j, 2); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pTab->aCol[pFK->aCol[j].iFrom].zName, 0); + sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } + ++i; + pFK = pFK->pNextFrom; } - regRecord = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nCol); - sqlite3WhereEnd(pWInfo); - sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); - VdbeComment((v, "GROUP BY sort")); - sAggInfo.useSortingIdx = 1; - sqlite3ExprCacheClear(pParse); } + } + }else +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ - /* Evaluate the current GROUP BY terms and store in b0, b1, b2... - ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth) - ** Then compare the current GROUP BY terms against the GROUP BY terms - ** from the previous row currently stored in a0, a1, a2... - */ - addrTopOfLoop = sqlite3VdbeCurrentAddr(v); - sqlite3ExprCacheClear(pParse); - for(j=0; jnExpr; j++){ - if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); - }else{ - sAggInfo.directMode = 1; - sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); - } +#ifndef NDEBUG + if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ + if( zRight ){ + if( sqlite3GetBoolean(zRight) ){ + sqlite3ParserTrace(stderr, "parser: "); + }else{ + sqlite3ParserTrace(0, 0); } - sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, - (char*)pKeyInfo, P4_KEYINFO); - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); + } + }else +#endif - /* Generate code that runs whenever the GROUP BY changes. - ** Changes in the GROUP BY are detected by the previous code - ** block. If there were no changes, this block is skipped. - ** - ** This code copies current group by terms in b0,b1,b2,... - ** over to a0,a1,a2. It then calls the output subroutine - ** and resets the aggregate accumulator registers in preparation - ** for the next GROUP BY batch. - */ - sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output one row")); - sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); - VdbeComment((v, "check abort flag")); - sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - VdbeComment((v, "reset accumulator")); + /* Reinstall the LIKE and GLOB functions. The variant of LIKE + ** used will be case sensitive or not depending on the RHS. + */ + if( sqlite3StrICmp(zLeft, "case_sensitive_like")==0 ){ + if( zRight ){ + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight)); + } + }else - /* Update the aggregate accumulators based on the content of - ** the current row - */ - sqlite3VdbeJumpHere(v, j1); - updateAccumulator(pParse, &sAggInfo); - sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); - VdbeComment((v, "indicate data in accumulator")); +#ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX +# define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 +#endif - /* End of the loop - */ - if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); - }else{ - sqlite3WhereEnd(pWInfo); - sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + /* Pragma "quick_check" is an experimental reduced version of + ** integrity_check designed to detect most database corruption + ** without most of the overhead of a full integrity-check. + */ + if( sqlite3StrICmp(zLeft, "integrity_check")==0 + || sqlite3StrICmp(zLeft, "quick_check")==0 + ){ + int i, j, addr, mxErr; + + /* Code that appears at the end of the integrity check. If no error + ** messages have been generated, output OK. Otherwise output the + ** error message + */ + static const VdbeOpList endCode[] = { + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_IfNeg, 1, 0, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, + }; + + int isQuick = (zLeft[0]=='q'); + + /* Initialize the VDBE program */ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + pParse->nMem = 6; + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); + + /* Set the maximum error count */ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + if( zRight ){ + sqlite3GetInt32(zRight, &mxErr); + if( mxErr<=0 ){ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } + } + sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ - /* Output the final row of result - */ - sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output final row")); + /* Do an integrity check on each database file */ + for(i=0; inDb; i++){ + HashElem *x; + Hash *pTbls; + int cnt = 0; - /* Jump over the subroutines - */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + if( OMIT_TEMPDB && i==1 ) continue; - /* Generate a subroutine that outputs a single row of the result - ** set. This subroutine first looks at the iUseFlag. If iUseFlag - ** is less than or equal to zero, the subroutine is a no-op. If - ** the processing calls for the query to abort, this subroutine - ** increments the iAbortFlag memory location before returning in - ** order to signal the caller to abort. + sqlite3CodeVerifySchema(pParse, i); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeJumpHere(v, addr); + + /* Do an integrity check of the B-Tree + ** + ** Begin by filling registers 2, 3, ... with the root pages numbers + ** for all tables and indices in the database. */ - addrSetAbort = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag); - VdbeComment((v, "set abort flag")); - sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - sqlite3VdbeResolveLabel(v, addrOutputRow); - addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); - VdbeComment((v, "Groupby result generator entry point")); - sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - finalizeAggFunctions(pParse, &sAggInfo); - sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, - distinct, pDest, - addrOutputRow+1, addrSetAbort); - sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - VdbeComment((v, "end groupby result generator")); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pTbls = &db->aDb[i].pSchema->tblHash; + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); + cnt++; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); + cnt++; + } + } - /* Generate a subroutine that will reset the group-by accumulator + /* Make sure sufficient number of registers have been allocated */ + if( pParse->nMem < cnt+4 ){ + pParse->nMem = cnt+4; + } + + /* Do the b-tree integrity checks */ + sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeChangeP5(v, (u8)i); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); + sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); + sqlite3VdbeJumpHere(v, addr); + + /* Make sure all the indices are constructed correctly. */ - sqlite3VdbeResolveLabel(v, addrReset); - resetAccumulator(pParse, &sAggInfo); - sqlite3VdbeAddOp1(v, OP_Return, regReset); - - } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ - else { - ExprList *pDel = 0; -#ifndef SQLITE_OMIT_BTREECOUNT - Table *pTab; - if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ - /* If isSimpleCount() returns a pointer to a Table structure, then - ** the SQL statement is of the form: - ** - ** SELECT count(*) FROM - ** - ** where the Table structure returned represents table . - ** - ** This statement is so common that it is optimized specially. The - ** OP_Count instruction is executed either on the intkey table that - ** contains the data for table or on one of its indexes. It - ** is better to execute the op on an index, as indexes are almost - ** always spread across less pages than their corresponding tables. - */ - const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */ - Index *pIdx; /* Iterator variable */ - KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */ - Index *pBest = 0; /* Best index found so far */ - int iRoot = pTab->tnum; /* Root page of scanned b-tree */ - - sqlite3CodeVerifySchema(pParse, iDb); - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + int loopTop; - /* Search for the index that has the least amount of columns. If - ** there is such an index, and it has less columns than the table - ** does, then we can assume that it consumes less space on disk and - ** will therefore be cheaper to scan to determine the query result. - ** In this case set iRoot to the root page number of the index b-tree - ** and pKeyInfo to the KeyInfo structure required to navigate the - ** index. - ** - ** In practice the KeyInfo structure will not be used. It is only - ** passed to keep OP_OpenRead happy. - */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pBest || pIdx->nColumnnColumn ){ - pBest = pIdx; - } + if( pTab->pIndex==0 ) continue; + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeJumpHere(v, addr); + sqlite3OpenTableAndIndices(pParse, pTab, 1, OP_OpenRead); + sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */ + loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0); + sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */ + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int jmp2; + int r1; + static const VdbeOpList idxErr[] = { + { OP_AddImm, 1, -1, 0}, + { OP_String8, 0, 3, 0}, /* 1 */ + { OP_Rowid, 1, 4, 0}, + { OP_String8, 0, 5, 0}, /* 3 */ + { OP_String8, 0, 6, 0}, /* 4 */ + { OP_Concat, 4, 3, 3}, + { OP_Concat, 5, 3, 3}, + { OP_Concat, 6, 3, 3}, + { OP_ResultRow, 3, 1, 0}, + { OP_IfPos, 1, 0, 0}, /* 9 */ + { OP_Halt, 0, 0, 0}, + }; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0); + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1); + addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr); + sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC); + sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC); + sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeJumpHere(v, addr+9); + sqlite3VdbeJumpHere(v, jmp2); } - if( pBest && pBest->nColumnnCol ){ - iRoot = pBest->tnum; - pKeyInfo = sqlite3IndexKeyinfo(pParse, pBest); + sqlite3VdbeAddOp2(v, OP_Next, 1, loopTop+1); + sqlite3VdbeJumpHere(v, loopTop); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + static const VdbeOpList cntIdx[] = { + { OP_Integer, 0, 3, 0}, + { OP_Rewind, 0, 0, 0}, /* 1 */ + { OP_AddImm, 3, 1, 0}, + { OP_Next, 0, 0, 0}, /* 3 */ + { OP_Eq, 2, 0, 3}, /* 4 */ + { OP_AddImm, 1, -1, 0}, + { OP_String8, 0, 2, 0}, /* 6 */ + { OP_String8, 0, 3, 0}, /* 7 */ + { OP_Concat, 3, 2, 2}, + { OP_ResultRow, 2, 1, 0}, + }; + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + sqlite3VdbeJumpHere(v, addr); + addr = sqlite3VdbeAddOpList(v, ArraySize(cntIdx), cntIdx); + sqlite3VdbeChangeP1(v, addr+1, j+2); + sqlite3VdbeChangeP2(v, addr+1, addr+4); + sqlite3VdbeChangeP1(v, addr+3, j+2); + sqlite3VdbeChangeP2(v, addr+3, addr+2); + sqlite3VdbeJumpHere(v, addr+4); + sqlite3VdbeChangeP4(v, addr+6, + "wrong # of entries in index ", P4_STATIC); + sqlite3VdbeChangeP4(v, addr+7, pIdx->zName, P4_TRANSIENT); } + } + } + addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode); + sqlite3VdbeChangeP2(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr+1); + sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); + }else +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ - /* Open a read-only cursor, execute the OP_Count, close the cursor. */ - sqlite3VdbeAddOp3(v, OP_OpenRead, iCsr, iRoot, iDb); - if( pKeyInfo ){ - sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO_HANDOFF); - } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); - sqlite3VdbeAddOp1(v, OP_Close, iCsr); - }else -#endif /* SQLITE_OMIT_BTREECOUNT */ - { - /* Check if the query is of one of the following forms: - ** - ** SELECT min(x) FROM ... - ** SELECT max(x) FROM ... - ** - ** If it is, then ask the code in where.c to attempt to sort results - ** as if there was an "ORDER ON x" or "ORDER ON x DESC" clause. - ** If where.c is able to produce results sorted in this order, then - ** add vdbe code to break out of the processing loop after the - ** first iteration (since the first iteration of the loop is - ** guaranteed to operate on the row with the minimum or maximum - ** value of x, the only row required). - ** - ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: - ** - ** + If the query is a "SELECT min(x)", then the loop coded by - ** where.c should not iterate over any values with a NULL value - ** for x. - ** - ** + The optimizer code in where.c (the thing that decides which - ** index or indices to use) should place a different priority on - ** satisfying the 'ORDER BY' clause than it does in other cases. - ** Refer to code and comments in where.c for details. - */ - ExprList *pMinMax = 0; - u8 flag = minMaxQuery(p); - if( flag ){ - assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); - pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); - pDel = pMinMax; - if( pMinMax && !db->mallocFailed ){ - pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; - pMinMax->a[0].pExpr->op = TK_COLUMN; +#ifndef SQLITE_OMIT_UTF16 + /* + ** PRAGMA encoding + ** PRAGMA encoding = "utf-8"|"utf-16"|"utf-16le"|"utf-16be" + ** + ** In its first form, this pragma returns the encoding of the main + ** database. If the database is not initialized, it is initialized now. + ** + ** The second form of this pragma is a no-op if the main database file + ** has not already been initialized. In this case it sets the default + ** encoding that will be used for the main database file if a new file + ** is created. If an existing main database file is opened, then the + ** default text encoding for the existing database is used. + ** + ** In all cases new databases created using the ATTACH command are + ** created to use the same default text encoding as the main database. If + ** the main database has not been initialized and/or created when ATTACH + ** is executed, this is done before the ATTACH operation. + ** + ** In the second form this pragma sets the text encoding to be used in + ** new database files created using this database handle. It is only + ** useful if invoked immediately after the main database i + */ + if( sqlite3StrICmp(zLeft, "encoding")==0 ){ + static const struct EncName { + char *zName; + u8 enc; + } encnames[] = { + { "UTF8", SQLITE_UTF8 }, + { "UTF-8", SQLITE_UTF8 }, /* Must be element [1] */ + { "UTF-16le", SQLITE_UTF16LE }, /* Must be element [2] */ + { "UTF-16be", SQLITE_UTF16BE }, /* Must be element [3] */ + { "UTF16le", SQLITE_UTF16LE }, + { "UTF16be", SQLITE_UTF16BE }, + { "UTF-16", 0 }, /* SQLITE_UTF16NATIVE */ + { "UTF16", 0 }, /* SQLITE_UTF16NATIVE */ + { 0, 0 } + }; + const struct EncName *pEnc; + if( !zRight ){ /* "PRAGMA encoding" */ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_String8, 0, 1); + assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); + assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); + assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); + sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + }else{ /* "PRAGMA encoding = XXX" */ + /* Only change the value of sqlite.enc if the database handle is not + ** initialized. If the main database exists, the new sqlite.enc value + ** will be overwritten when the schema is next loaded. If it does not + ** already exists, it will be created to use the new encoding value. + */ + if( + !(DbHasProperty(db, 0, DB_SchemaLoaded)) || + DbHasProperty(db, 0, DB_Empty) + ){ + for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ + if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ + ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + break; } } - - /* This case runs if the aggregate has no GROUP BY clause. The - ** processing is much simpler since there is only a single row - ** of output. - */ - resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); - if( pWInfo==0 ){ - sqlite3ExprListDelete(db, pDel); - goto select_end; - } - updateAccumulator(pParse, &sAggInfo); - if( !pMinMax && flag ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); - VdbeComment((v, "%s() by index", - (flag==WHERE_ORDERBY_MIN?"min":"max"))); + if( !pEnc->zName ){ + sqlite3ErrorMsg(pParse, "unsupported encoding: %s", zRight); } - sqlite3WhereEnd(pWInfo); - finalizeAggFunctions(pParse, &sAggInfo); } + } + }else +#endif /* SQLITE_OMIT_UTF16 */ - pOrderBy = 0; - sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, - pDest, addrEnd, addrEnd); - sqlite3ExprListDelete(db, pDel); +#ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + /* + ** PRAGMA [database.]schema_version + ** PRAGMA [database.]schema_version = + ** + ** PRAGMA [database.]user_version + ** PRAGMA [database.]user_version = + ** + ** The pragma's schema_version and user_version are used to set or get + ** the value of the schema-version and user-version, respectively. Both + ** the schema-version and the user-version are 32-bit signed integers + ** stored in the database header. + ** + ** The schema-cookie is usually only manipulated internally by SQLite. It + ** is incremented by SQLite whenever the database schema is modified (by + ** creating or dropping a table or index). The schema version is used by + ** SQLite each time a query is executed to ensure that the internal cache + ** of the schema used when compiling the SQL query matches the schema of + ** the database against which the compiled query is actually executed. + ** Subverting this mechanism by using "PRAGMA schema_version" to modify + ** the schema-version is potentially dangerous and may lead to program + ** crashes or database corruption. Use with caution! + ** + ** The user-version is not used internally by SQLite. It may be used by + ** applications for any purpose. + */ + if( sqlite3StrICmp(zLeft, "schema_version")==0 + || sqlite3StrICmp(zLeft, "user_version")==0 + || sqlite3StrICmp(zLeft, "freelist_count")==0 + ){ + int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ + sqlite3VdbeUsesBtree(v, iDb); + switch( zLeft[0] ){ + case 'f': case 'F': + iCookie = BTREE_FREE_PAGE_COUNT; + break; + case 's': case 'S': + iCookie = BTREE_SCHEMA_VERSION; + break; + default: + iCookie = BTREE_USER_VERSION; + break; } - sqlite3VdbeResolveLabel(v, addrEnd); - - } /* endif aggregate query */ - if( distinct>=0 ){ - explainTempTable(pParse, "DISTINCT"); - } + if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ + /* Write the specified cookie value */ + static const VdbeOpList setCookie[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_Integer, 0, 1, 0}, /* 1 */ + { OP_SetCookie, 0, 0, 1}, /* 2 */ + }; + int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); + sqlite3VdbeChangeP1(v, addr+2, iDb); + sqlite3VdbeChangeP2(v, addr+2, iCookie); + }else{ + /* Read the specified cookie value */ + static const VdbeOpList readCookie[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, 0}, /* 1 */ + { OP_ResultRow, 1, 1, 0} + }; + int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie); + sqlite3VdbeChangeP1(v, addr, iDb); + sqlite3VdbeChangeP1(v, addr+1, iDb); + sqlite3VdbeChangeP3(v, addr+1, iCookie); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + } + }else +#endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ - /* If there is an ORDER BY clause, then we need to sort the results - ** and send them to the callback one by one. +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + /* + ** PRAGMA compile_options + ** + ** Return the names of all compile-time options used in this build, + ** one option per row. */ - if( pOrderBy ){ - explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, v, pEList->nExpr, pDest); - } + if( sqlite3StrICmp(zLeft, "compile_options")==0 ){ + int i = 0; + const char *zOpt; + sqlite3VdbeSetNumCols(v, 1); + pParse->nMem = 1; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ - /* Jump here to skip this query +#ifndef SQLITE_OMIT_WAL + /* + ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** + ** Checkpoint the database. */ - sqlite3VdbeResolveLabel(v, iEnd); + if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){ + int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); + int eMode = SQLITE_CHECKPOINT_PASSIVE; + if( zRight ){ + if( sqlite3StrICmp(zRight, "full")==0 ){ + eMode = SQLITE_CHECKPOINT_FULL; + }else if( sqlite3StrICmp(zRight, "restart")==0 ){ + eMode = SQLITE_CHECKPOINT_RESTART; + } + } + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); - /* The SELECT was successfully coded. Set the return code to 0 - ** to indicate no errors. + sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + }else + + /* + ** PRAGMA wal_autocheckpoint + ** PRAGMA wal_autocheckpoint = N + ** + ** Configure a database connection to automatically checkpoint a database + ** after accumulating N frames in the log. Or query for the current value + ** of N. */ - rc = 0; + if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){ + if( zRight ){ + sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); + } + returnSingleInt(pParse, "wal_autocheckpoint", + db->xWalCallback==sqlite3WalDefaultHook ? + SQLITE_PTR_TO_INT(db->pWalArg) : 0); + }else +#endif - /* Control jumps to here if an error is encountered above, or upon - ** successful coding of the SELECT. +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* + ** Report the current state of file logs for all databases */ -select_end: - explainSetInteger(pParse->iSelectId, iRestoreSelectId); + if( sqlite3StrICmp(zLeft, "lock_status")==0 ){ + static const char *const azLockName[] = { + "unlocked", "shared", "reserved", "pending", "exclusive" + }; + int i; + sqlite3VdbeSetNumCols(v, 2); + pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); + for(i=0; inDb; i++){ + Btree *pBt; + Pager *pPager; + const char *zState = "unknown"; + int j; + if( db->aDb[i].zName==0 ) continue; + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); + pBt = db->aDb[i].pBt; + if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ + zState = "closed"; + }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ + zState = azLockName[j]; + } + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); + } - /* Identify column names if results of the SELECT are to be output. + }else +#endif + +#ifdef SQLITE_HAS_CODEC + if( sqlite3StrICmp(zLeft, "key")==0 && zRight ){ + sqlite3_key(db, zRight, sqlite3Strlen30(zRight)); + }else + if( sqlite3StrICmp(zLeft, "rekey")==0 && zRight ){ + sqlite3_rekey(db, zRight, sqlite3Strlen30(zRight)); + }else + if( zRight && (sqlite3StrICmp(zLeft, "hexkey")==0 || + sqlite3StrICmp(zLeft, "hexrekey")==0) ){ + int i, h1, h2; + char zKey[40]; + for(i=0; (h1 = zRight[i])!=0 && (h2 = zRight[i+1])!=0; i+=2){ + h1 += 9*(1&(h1>>6)); + h2 += 9*(1&(h2>>6)); + zKey[i/2] = (h2 & 0x0f) | ((h1 & 0xf)<<4); + } + if( (zLeft[3] & 0xf)==0xb ){ + sqlite3_key(db, zKey, i/2); + }else{ + sqlite3_rekey(db, zKey, i/2); + } + }else +#endif +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) + if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ +#ifdef SQLITE_HAS_CODEC + if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ + sqlite3_activate_see(&zRight[4]); + } +#endif +#ifdef SQLITE_ENABLE_CEROD + if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){ + sqlite3_activate_cerod(&zRight[6]); + } +#endif + }else +#endif + + + {/* Empty ELSE clause */} + + /* + ** Reset the safety level, in case the fullfsync flag or synchronous + ** setting changed. */ - if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ - generateColumnNames(pParse, pTabList, pEList); +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + if( db->autoCommit ){ + sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, + (db->flags&SQLITE_FullFSync)!=0, + (db->flags&SQLITE_CkptFullFSync)!=0); } - - sqlite3DbFree(db, sAggInfo.aCol); - sqlite3DbFree(db, sAggInfo.aFunc); - return rc; +#endif +pragma_out: + sqlite3DbFree(db, zLeft); + sqlite3DbFree(db, zRight); } -#if defined(SQLITE_DEBUG) +#endif /* SQLITE_OMIT_PRAGMA */ + +/************** End of pragma.c **********************************************/ +/************** Begin file prepare.c *****************************************/ /* -******************************************************************************* -** The following code is used for testing and debugging only. The code -** that follows does not appear in normal builds. +** 2005 May 25 ** -** These routines are used to print out the content of all or part of a -** parse structures such as Select or Expr. Such printouts are useful -** for helping to understand what is happening inside the code generator -** during the execution of complex SELECT statements. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** These routine are not called anywhere from within the normal -** code base. Then are intended to be called from within the debugger -** or from temporary "printf" statements inserted for debugging. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_prepare() +** interface, and routines that contribute to loading the database schema +** from disk. */ -SQLITE_PRIVATE void sqlite3PrintExpr(Expr *p){ - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - sqlite3DebugPrintf("(%s", p->u.zToken); - }else{ - sqlite3DebugPrintf("(%d", p->op); - } - if( p->pLeft ){ - sqlite3DebugPrintf(" "); - sqlite3PrintExpr(p->pLeft); - } - if( p->pRight ){ - sqlite3DebugPrintf(" "); - sqlite3PrintExpr(p->pRight); - } - sqlite3DebugPrintf(")"); -} -SQLITE_PRIVATE void sqlite3PrintExprList(ExprList *pList){ - int i; - for(i=0; inExpr; i++){ - sqlite3PrintExpr(pList->a[i].pExpr); - if( inExpr-1 ){ - sqlite3DebugPrintf(", "); - } - } -} -SQLITE_PRIVATE void sqlite3PrintSelect(Select *p, int indent){ - sqlite3DebugPrintf("%*sSELECT(%p) ", indent, "", p); - sqlite3PrintExprList(p->pEList); - sqlite3DebugPrintf("\n"); - if( p->pSrc ){ - char *zPrefix; - int i; - zPrefix = "FROM"; - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - sqlite3DebugPrintf("%*s ", indent+6, zPrefix); - zPrefix = ""; - if( pItem->pSelect ){ - sqlite3DebugPrintf("(\n"); - sqlite3PrintSelect(pItem->pSelect, indent+10); - sqlite3DebugPrintf("%*s)", indent+8, ""); - }else if( pItem->zName ){ - sqlite3DebugPrintf("%s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3DebugPrintf("(table: %s)", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3DebugPrintf(" AS %s", pItem->zAlias); - } - if( ipSrc->nSrc-1 ){ - sqlite3DebugPrintf(","); - } - sqlite3DebugPrintf("\n"); + +/* +** Fill the InitData structure with an error message that indicates +** that the database is corrupt. +*/ +static void corruptSchema( + InitData *pData, /* Initialization context */ + const char *zObj, /* Object being parsed at the point of error */ + const char *zExtra /* Error information */ +){ + sqlite3 *db = pData->db; + if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + if( zObj==0 ) zObj = "?"; + sqlite3SetString(pData->pzErrMsg, db, + "malformed database schema (%s)", zObj); + if( zExtra ){ + *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, + "%s - %s", *pData->pzErrMsg, zExtra); } } - if( p->pWhere ){ - sqlite3DebugPrintf("%*s WHERE ", indent, ""); - sqlite3PrintExpr(p->pWhere); - sqlite3DebugPrintf("\n"); - } - if( p->pGroupBy ){ - sqlite3DebugPrintf("%*s GROUP BY ", indent, ""); - sqlite3PrintExprList(p->pGroupBy); - sqlite3DebugPrintf("\n"); - } - if( p->pHaving ){ - sqlite3DebugPrintf("%*s HAVING ", indent, ""); - sqlite3PrintExpr(p->pHaving); - sqlite3DebugPrintf("\n"); - } - if( p->pOrderBy ){ - sqlite3DebugPrintf("%*s ORDER BY ", indent, ""); - sqlite3PrintExprList(p->pOrderBy); - sqlite3DebugPrintf("\n"); - } + pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; } -/* End of the structure debug printing code -*****************************************************************************/ -#endif /* defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ -/************** End of select.c **********************************************/ -/************** Begin file table.c *******************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** This is the callback routine for the code that initializes the +** database. See sqlite3Init() below for additional information. +** This routine is also called from the OP_ParseSchema opcode of the VDBE. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Each callback contains the following information: ** -************************************************************************* -** This file contains the sqlite3_get_table() and sqlite3_free_table() -** interface routines. These are just wrappers around the main -** interface routine of sqlite3_exec(). +** argv[0] = name of thing being created +** argv[1] = root page number for table or index. 0 for trigger or view. +** argv[2] = SQL text for the CREATE statement. ** -** These routines are in a separate files so that they will not be linked -** if they are not used. -*/ - -#ifndef SQLITE_OMIT_GET_TABLE - -/* -** This structure is used to pass data from sqlite3_get_table() through -** to the callback function is uses to build the result. -*/ -typedef struct TabResult { - char **azResult; /* Accumulated output */ - char *zErrMsg; /* Error message text, if an error occurs */ - int nAlloc; /* Slots allocated for azResult[] */ - int nRow; /* Number of rows in the result */ - int nColumn; /* Number of columns in the result */ - int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ - int rc; /* Return code from sqlite3_exec() */ -} TabResult; - -/* -** This routine is called once for each row in the result table. Its job -** is to fill in the TabResult structure appropriately, allocating new -** memory as necessary. */ -static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ - TabResult *p = (TabResult*)pArg; /* Result accumulator */ - int need; /* Slots needed in p->azResult[] */ - int i; /* Loop counter */ - char *z; /* A single column of result */ - - /* Make sure there is enough space in p->azResult to hold everything - ** we need to remember from this invocation of the callback. - */ - if( p->nRow==0 && argv!=0 ){ - need = nCol*2; - }else{ - need = nCol; - } - if( p->nData + need > p->nAlloc ){ - char **azNew; - p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); - if( azNew==0 ) goto malloc_failed; - p->azResult = azNew; - } +SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ + InitData *pData = (InitData*)pInit; + sqlite3 *db = pData->db; + int iDb = pData->iDb; - /* If this is the first row, then generate an extra row containing - ** the names of all columns. - */ - if( p->nRow==0 ){ - p->nColumn = nCol; - for(i=0; iazResult[p->nData++] = z; - } - }else if( p->nColumn!=nCol ){ - sqlite3_free(p->zErrMsg); - p->zErrMsg = sqlite3_mprintf( - "sqlite3_get_table() called with two or more incompatible queries" - ); - p->rc = SQLITE_ERROR; + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + assert( sqlite3_mutex_held(db->mutex) ); + DbClearProperty(db, iDb, DB_Empty); + if( db->mallocFailed ){ + corruptSchema(pData, argv[0], 0); return 1; } - /* Copy over the row data - */ - if( argv!=0 ){ - for(i=0; i=0 && iDbnDb ); + if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ + if( argv[1]==0 ){ + corruptSchema(pData, argv[0], 0); + }else if( argv[2] && argv[2][0] ){ + /* Call the parser to process a CREATE TABLE, INDEX or VIEW. + ** But because db->init.busy is set to 1, no VDBE code is generated + ** or executed. All the parser does is build the internal data + ** structures that describe the table, index, or view. + */ + int rc; + sqlite3_stmt *pStmt; + TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ + + assert( db->init.busy ); + db->init.iDb = iDb; + db->init.newTnum = sqlite3Atoi(argv[1]); + db->init.orphanTrigger = 0; + TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); + rc = db->errCode; + assert( (rc&0xFF)==(rcp&0xFF) ); + db->init.iDb = 0; + if( SQLITE_OK!=rc ){ + if( db->init.orphanTrigger ){ + assert( iDb==1 ); }else{ - int n = sqlite3Strlen30(argv[i])+1; - z = sqlite3_malloc( n ); - if( z==0 ) goto malloc_failed; - memcpy(z, argv[i], n); + pData->rc = rc; + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ + corruptSchema(pData, argv[0], sqlite3_errmsg(db)); + } } - p->azResult[p->nData++] = z; } - p->nRow++; + sqlite3_finalize(pStmt); + }else if( argv[0]==0 ){ + corruptSchema(pData, 0, 0); + }else{ + /* If the SQL column is blank it means this is an index that + ** was created to be the PRIMARY KEY or to fulfill a UNIQUE + ** constraint for a CREATE TABLE. The index should have already + ** been created when we processed the CREATE TABLE. All we have + ** to do here is record the root page number for that index. + */ + Index *pIndex; + pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); + if( pIndex==0 ){ + /* This can occur if there exists an index on a TEMP table which + ** has the same name as another index on a permanent index. Since + ** the permanent table is hidden by the TEMP table, we can also + ** safely ignore the index on the permanent table. + */ + /* Do Nothing */; + }else if( sqlite3GetInt32(argv[1], &pIndex->tnum)==0 ){ + corruptSchema(pData, argv[0], "invalid rootpage"); + } } return 0; - -malloc_failed: - p->rc = SQLITE_NOMEM; - return 1; } /* -** Query the database. But instead of invoking a callback for each row, -** malloc() for space to hold the result and return the entire results -** at the conclusion of the call. -** -** The result that is written to ***pazResult is held in memory obtained -** from malloc(). But the caller cannot free this memory directly. -** Instead, the entire table should be passed to sqlite3_free_table() when -** the calling procedure is finished using it. +** Attempt to read the database schema and initialize internal +** data structures for a single database file. The index of the +** database file is given by iDb. iDb==0 is used for the main +** database. iDb==1 should never be used. iDb>=2 is used for +** auxiliary databases. Return one of the SQLITE_ error codes to +** indicate success or failure. */ -SQLITE_API int sqlite3_get_table( - sqlite3 *db, /* The database on which the SQL executes */ - const char *zSql, /* The SQL to be executed */ - char ***pazResult, /* Write the result table here */ - int *pnRow, /* Write the number of rows in the result here */ - int *pnColumn, /* Write the number of columns of result here */ - char **pzErrMsg /* Write error messages here */ -){ +static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ int rc; - TabResult res; + int i; + int size; + Table *pTab; + Db *pDb; + char const *azArg[4]; + int meta[5]; + InitData initData; + char const *zMasterSchema; + char const *zMasterName; + int openedTransaction = 0; - *pazResult = 0; - if( pnColumn ) *pnColumn = 0; - if( pnRow ) *pnRow = 0; - if( pzErrMsg ) *pzErrMsg = 0; - res.zErrMsg = 0; - res.nRow = 0; - res.nColumn = 0; - res.nData = 1; - res.nAlloc = 20; - res.rc = SQLITE_OK; - res.azResult = sqlite3_malloc(sizeof(char*)*res.nAlloc ); - if( res.azResult==0 ){ - db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM; + /* + ** The master database table has a structure like this + */ + static const char master_schema[] = + "CREATE TABLE sqlite_master(\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")" + ; +#ifndef SQLITE_OMIT_TEMPDB + static const char temp_master_schema[] = + "CREATE TEMP TABLE sqlite_temp_master(\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")" + ; +#else + #define temp_master_schema 0 +#endif + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + + /* zMasterSchema and zInitScript are set to point at the master schema + ** and initialisation script appropriate for the database being + ** initialised. zMasterName is the name of the master table. + */ + if( !OMIT_TEMPDB && iDb==1 ){ + zMasterSchema = temp_master_schema; + }else{ + zMasterSchema = master_schema; } - res.azResult[0] = 0; - rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); - assert( sizeof(res.azResult[0])>= sizeof(res.nData) ); - res.azResult[0] = SQLITE_INT_TO_PTR(res.nData); - if( (rc&0xff)==SQLITE_ABORT ){ - sqlite3_free_table(&res.azResult[1]); - if( res.zErrMsg ){ - if( pzErrMsg ){ - sqlite3_free(*pzErrMsg); - *pzErrMsg = sqlite3_mprintf("%s",res.zErrMsg); - } - sqlite3_free(res.zErrMsg); - } - db->errCode = res.rc; /* Assume 32-bit assignment is atomic */ - return res.rc; + zMasterName = SCHEMA_TABLE(iDb); + + /* Construct the schema tables. */ + azArg[0] = zMasterName; + azArg[1] = "1"; + azArg[2] = zMasterSchema; + azArg[3] = 0; + initData.db = db; + initData.iDb = iDb; + initData.rc = SQLITE_OK; + initData.pzErrMsg = pzErrMsg; + sqlite3InitCallback(&initData, 3, (char **)azArg, 0); + if( initData.rc ){ + rc = initData.rc; + goto error_out; } - sqlite3_free(res.zErrMsg); - if( rc!=SQLITE_OK ){ - sqlite3_free_table(&res.azResult[1]); - return rc; + pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); + if( ALWAYS(pTab) ){ + pTab->tabFlags |= TF_Readonly; } - if( res.nAlloc>res.nData ){ - char **azNew; - azNew = sqlite3_realloc( res.azResult, sizeof(char*)*res.nData ); - if( azNew==0 ){ - sqlite3_free_table(&res.azResult[1]); - db->errCode = SQLITE_NOMEM; - return SQLITE_NOMEM; + + /* Create a cursor to hold the database open + */ + pDb = &db->aDb[iDb]; + if( pDb->pBt==0 ){ + if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){ + DbSetProperty(db, 1, DB_SchemaLoaded); } - res.azResult = azNew; + return SQLITE_OK; } - *pazResult = &res.azResult[1]; - if( pnColumn ) *pnColumn = res.nColumn; - if( pnRow ) *pnRow = res.nRow; - return rc; -} -/* -** This routine frees the space the sqlite3_get_table() malloced. -*/ -SQLITE_API void sqlite3_free_table( - char **azResult /* Result returned from from sqlite3_get_table() */ -){ - if( azResult ){ - int i, n; - azResult--; - assert( azResult!=0 ); - n = SQLITE_PTR_TO_INT(azResult[0]); - for(i=1; ipBt); + if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); + if( rc!=SQLITE_OK ){ + sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto initone_error_out; + } + openedTransaction = 1; } -} - -#endif /* SQLITE_OMIT_GET_TABLE */ -/************** End of table.c ***********************************************/ -/************** Begin file trigger.c *****************************************/ -/* -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains the implementation for TRIGGERs -*/ + /* Get the database meta information. + ** + ** Meta values are as follows: + ** meta[0] Schema cookie. Changes with each schema change. + ** meta[1] File format of schema layer. + ** meta[2] Size of the page cache. + ** meta[3] Largest rootpage (auto/incr_vacuum mode) + ** meta[4] Db text encoding. 1:UTF-8 2:UTF-16LE 3:UTF-16BE + ** meta[5] User version + ** meta[6] Incremental vacuum mode + ** meta[7] unused + ** meta[8] unused + ** meta[9] unused + ** + ** Note: The #defined SQLITE_UTF* symbols in sqliteInt.h correspond to + ** the possible values of meta[4]. + */ + for(i=0; ipBt, i+1, (u32 *)&meta[i]); + } + pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1]; -#ifndef SQLITE_OMIT_TRIGGER -/* -** Delete a linked list of TriggerStep structures. -*/ -SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ - while( pTriggerStep ){ - TriggerStep * pTmp = pTriggerStep; - pTriggerStep = pTriggerStep->pNext; + /* If opening a non-empty database, check the text encoding. For the + ** main database, set sqlite3.enc to the encoding of the main database. + ** For an attached db, it is an error if the encoding is not the same + ** as sqlite3.enc. + */ + if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ + if( iDb==0 ){ + u8 encoding; + /* If opening the main database, set ENC(db). */ + encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; + if( encoding==0 ) encoding = SQLITE_UTF8; + ENC(db) = encoding; + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + }else{ + /* If opening an attached database, the encoding much match ENC(db) */ + if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ + sqlite3SetString(pzErrMsg, db, "attached databases must use the same" + " text encoding as main database"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + } + }else{ + DbSetProperty(db, iDb, DB_Empty); + } + pDb->pSchema->enc = ENC(db); - sqlite3ExprDelete(db, pTmp->pWhere); - sqlite3ExprListDelete(db, pTmp->pExprList); - sqlite3SelectDelete(db, pTmp->pSelect); - sqlite3IdListDelete(db, pTmp->pIdList); + if( pDb->pSchema->cache_size==0 ){ + size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]); + if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } - sqlite3DbFree(db, pTmp); + /* + ** file_format==1 Version 3.0.0. + ** file_format==2 Version 3.1.3. // ALTER TABLE ADD COLUMN + ** file_format==3 Version 3.1.4. // ditto but with non-NULL defaults + ** file_format==4 Version 3.3.0. // DESC indices. Boolean constants + */ + pDb->pSchema->file_format = (u8)meta[BTREE_FILE_FORMAT-1]; + if( pDb->pSchema->file_format==0 ){ + pDb->pSchema->file_format = 1; + } + if( pDb->pSchema->file_format>SQLITE_MAX_FILE_FORMAT ){ + sqlite3SetString(pzErrMsg, db, "unsupported file format"); + rc = SQLITE_ERROR; + goto initone_error_out; } -} -/* -** Given table pTab, return a list of all the triggers attached to -** the table. The list is connected by Trigger.pNext pointers. -** -** All of the triggers on pTab that are in the same database as pTab -** are already attached to pTab->pTrigger. But there might be additional -** triggers on pTab in the TEMP schema. This routine prepends all -** TEMP triggers on pTab to the beginning of the pTab->pTrigger list -** and returns the combined list. -** -** To state it another way: This routine returns a list of all triggers -** that fire off of pTab. The list will include any TEMP triggers on -** pTab as well as the triggers lised in pTab->pTrigger. -*/ -SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ - Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; - Trigger *pList = 0; /* List of triggers to return */ - - if( pParse->disableTriggers ){ - return 0; - } - - if( pTmpSchema!=pTab->pSchema ){ - HashElem *p; - for(p=sqliteHashFirst(&pTmpSchema->trigHash); p; p=sqliteHashNext(p)){ - Trigger *pTrig = (Trigger *)sqliteHashData(p); - if( pTrig->pTabSchema==pTab->pSchema - && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - ){ - pTrig->pNext = (pList ? pList : pTab->pTrigger); - pList = pTrig; - } - } + /* Ticket #2804: When we open a database in the newer file format, + ** clear the legacy_file_format pragma flag so that a VACUUM will + ** not downgrade the database and thus invalidate any descending + ** indices that the user might have created. + */ + if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ + db->flags &= ~SQLITE_LegacyFileFmt; } - return (pList ? pList : pTab->pTrigger); -} - -/* -** This is called by the parser when it sees a CREATE TRIGGER statement -** up to the point of the BEGIN before the trigger actions. A Trigger -** structure is generated based on the information available and stored -** in pParse->pNewTrigger. After the trigger actions have been parsed, the -** sqlite3FinishTrigger() function is called to complete the trigger -** construction process. -*/ -SQLITE_PRIVATE void sqlite3BeginTrigger( - Parse *pParse, /* The parse context of the CREATE TRIGGER statement */ - Token *pName1, /* The name of the trigger */ - Token *pName2, /* The name of the trigger */ - int tr_tm, /* One of TK_BEFORE, TK_AFTER, TK_INSTEAD */ - int op, /* One of TK_INSERT, TK_UPDATE, TK_DELETE */ - IdList *pColumns, /* column list if this is an UPDATE OF trigger */ - SrcList *pTableName,/* The name of the table/view the trigger applies to */ - Expr *pWhen, /* WHEN clause */ - int isTemp, /* True if the TEMPORARY keyword is present */ - int noErr /* Suppress errors if the trigger already exists */ -){ - Trigger *pTrigger = 0; /* The new trigger */ - Table *pTab; /* Table that the trigger fires off of */ - char *zName = 0; /* Name of the trigger */ - sqlite3 *db = pParse->db; /* The database connection */ - int iDb; /* The database to store the trigger in */ - Token *pName; /* The unqualified db name */ - DbFixer sFix; /* State vector for the DB fixer */ - int iTabDb; /* Index of the database holding pTab */ - - assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ - assert( pName2!=0 ); - assert( op==TK_INSERT || op==TK_UPDATE || op==TK_DELETE ); - assert( op>0 && op<0xff ); - if( isTemp ){ - /* If TEMP was specified, then the trigger name may not be qualified. */ - if( pName2->n>0 ){ - sqlite3ErrorMsg(pParse, "temporary trigger may not have qualified name"); - goto trigger_cleanup; + /* Read the schema information out of the schema tables + */ + assert( db->init.busy ); + { + char *zSql; + zSql = sqlite3MPrintf(db, + "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", + db->aDb[iDb].zName, zMasterName); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + xAuth = db->xAuth; + db->xAuth = 0; +#endif + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; } - iDb = 1; - pName = pName1; - }else{ - /* Figure out the db that the the trigger will be created in */ - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); - if( iDb<0 ){ - goto trigger_cleanup; +#endif + if( rc==SQLITE_OK ) rc = initData.rc; + sqlite3DbFree(db, zSql); +#ifndef SQLITE_OMIT_ANALYZE + if( rc==SQLITE_OK ){ + sqlite3AnalysisLoad(db, iDb); } +#endif } - - /* If the trigger name was unqualified, and the table is a temp table, - ** then set iDb to 1 to create the trigger in the temporary database. - ** If sqlite3SrcListLookup() returns 0, indicating the table does not - ** exist, the error is caught by the block below. - */ - if( !pTableName || db->mallocFailed ){ - goto trigger_cleanup; + if( db->mallocFailed ){ + rc = SQLITE_NOMEM; + sqlite3ResetInternalSchema(db, -1); } - pTab = sqlite3SrcListLookup(pParse, pTableName); - if( db->init.busy==0 && pName2->n==0 && pTab - && pTab->pSchema==db->aDb[1].pSchema ){ - iDb = 1; + if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ + /* Black magic: If the SQLITE_RecoveryMode flag is set, then consider + ** the schema loaded, even if errors occurred. In this situation the + ** current sqlite3_prepare() operation will fail, but the following one + ** will attempt to compile the supplied statement against whatever subset + ** of the schema was loaded before the error occurred. The primary + ** purpose of this is to allow access to the sqlite_master table + ** even when its contents have been corrupted. + */ + DbSetProperty(db, iDb, DB_SchemaLoaded); + rc = SQLITE_OK; } - /* Ensure the table name matches database name and that the table exists */ - if( db->mallocFailed ) goto trigger_cleanup; - assert( pTableName->nSrc==1 ); - if( sqlite3FixInit(&sFix, pParse, iDb, "trigger", pName) && - sqlite3FixSrcList(&sFix, pTableName) ){ - goto trigger_cleanup; - } - pTab = sqlite3SrcListLookup(pParse, pTableName); - if( !pTab ){ - /* The table does not exist. */ - if( db->init.iDb==1 ){ - /* Ticket #3810. - ** Normally, whenever a table is dropped, all associated triggers are - ** dropped too. But if a TEMP trigger is created on a non-TEMP table - ** and the table is dropped by a different database connection, the - ** trigger is not visible to the database connection that does the - ** drop so the trigger cannot be dropped. This results in an - ** "orphaned trigger" - a trigger whose associated table is missing. - */ - db->init.orphanTrigger = 1; - } - goto trigger_cleanup; - } - if( IsVirtual(pTab) ){ - sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); - goto trigger_cleanup; + /* Jump here for an error that occurs after successfully allocating + ** curMain and calling sqlite3BtreeEnter(). For an error that occurs + ** before that point, jump to error_out. + */ +initone_error_out: + if( openedTransaction ){ + sqlite3BtreeCommit(pDb->pBt); } + sqlite3BtreeLeave(pDb->pBt); - /* Check that the trigger name is not reserved and that no trigger of the - ** specified name exists */ - zName = sqlite3NameFromToken(db, pName); - if( !zName || SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ - goto trigger_cleanup; - } - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), - zName, sqlite3Strlen30(zName)) ){ - if( !noErr ){ - sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); - } - goto trigger_cleanup; +error_out: + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; } + return rc; +} - /* Do not create a trigger on a system table */ - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ - sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); - pParse->nErr++; - goto trigger_cleanup; +/* +** Initialize all database files - the main database file, the file +** used to store temporary tables, and any additional database files +** created using ATTACH statements. Return a success code. If an +** error occurs, write an error message into *pzErrMsg. +** +** After a database is initialized, the DB_SchemaLoaded bit is set +** bit is set in the flags field of the Db structure. If the database +** file was of zero-length, then the DB_Empty flag is also set. +*/ +SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ + int i, rc; + int commit_internal = !(db->flags&SQLITE_InternChanges); + + assert( sqlite3_mutex_held(db->mutex) ); + rc = SQLITE_OK; + db->init.busy = 1; + for(i=0; rc==SQLITE_OK && inDb; i++){ + if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; + rc = sqlite3InitOne(db, i, pzErrMsg); + if( rc ){ + sqlite3ResetInternalSchema(db, i); + } } - /* INSTEAD of triggers are only for views and views only support INSTEAD - ** of triggers. + /* Once all the other databases have been initialised, load the schema + ** for the TEMP database. This is loaded last, as the TEMP database + ** schema may contain references to objects in other databases. */ - if( pTab->pSelect && tr_tm!=TK_INSTEAD ){ - sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", - (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName, 0); - goto trigger_cleanup; - } - if( !pTab->pSelect && tr_tm==TK_INSTEAD ){ - sqlite3ErrorMsg(pParse, "cannot create INSTEAD OF" - " trigger on table: %S", pTableName, 0); - goto trigger_cleanup; - } - iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); - -#ifndef SQLITE_OMIT_AUTHORIZATION - { - int code = SQLITE_CREATE_TRIGGER; - const char *zDb = db->aDb[iTabDb].zName; - const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb; - if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; - if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ - goto trigger_cleanup; - } - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iTabDb),0,zDb)){ - goto trigger_cleanup; +#ifndef SQLITE_OMIT_TEMPDB + if( rc==SQLITE_OK && ALWAYS(db->nDb>1) + && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, 1, pzErrMsg); + if( rc ){ + sqlite3ResetInternalSchema(db, 1); } } #endif - /* INSTEAD OF triggers can only appear on views and BEFORE triggers - ** cannot appear on views. So we might as well translate every - ** INSTEAD OF trigger into a BEFORE trigger. It simplifies code - ** elsewhere. - */ - if (tr_tm == TK_INSTEAD){ - tr_tm = TK_BEFORE; + db->init.busy = 0; + if( rc==SQLITE_OK && commit_internal ){ + sqlite3CommitInternalChanges(db); } - /* Build the Trigger object */ - pTrigger = (Trigger*)sqlite3DbMallocZero(db, sizeof(Trigger)); - if( pTrigger==0 ) goto trigger_cleanup; - pTrigger->zName = zName; - zName = 0; - pTrigger->table = sqlite3DbStrDup(db, pTableName->a[0].zName); - pTrigger->pSchema = db->aDb[iDb].pSchema; - pTrigger->pTabSchema = pTab->pSchema; - pTrigger->op = (u8)op; - pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; - pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); - pTrigger->pColumns = sqlite3IdListDup(db, pColumns); - assert( pParse->pNewTrigger==0 ); - pParse->pNewTrigger = pTrigger; - -trigger_cleanup: - sqlite3DbFree(db, zName); - sqlite3SrcListDelete(db, pTableName); - sqlite3IdListDelete(db, pColumns); - sqlite3ExprDelete(db, pWhen); - if( !pParse->pNewTrigger ){ - sqlite3DeleteTrigger(db, pTrigger); - }else{ - assert( pParse->pNewTrigger==pTrigger ); - } + return rc; } /* -** This routine is called after all of the trigger actions have been parsed -** in order to complete the process of building the trigger. +** This routine is a no-op if the database schema is already initialised. +** Otherwise, the schema is loaded. An error code is returned. */ -SQLITE_PRIVATE void sqlite3FinishTrigger( - Parse *pParse, /* Parser context */ - TriggerStep *pStepList, /* The triggered program */ - Token *pAll /* Token that describes the complete CREATE TRIGGER */ -){ - Trigger *pTrig = pParse->pNewTrigger; /* Trigger being finished */ - char *zName; /* Name of trigger */ - sqlite3 *db = pParse->db; /* The database */ - DbFixer sFix; /* Fixer object */ - int iDb; /* Database containing the trigger */ - Token nameToken; /* Trigger name for error reporting */ - - pTrig = pParse->pNewTrigger; - pParse->pNewTrigger = 0; - if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; - zName = pTrig->zName; - iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); - pTrig->step_list = pStepList; - while( pStepList ){ - pStepList->pTrig = pTrig; - pStepList = pStepList->pNext; +SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ + int rc = SQLITE_OK; + sqlite3 *db = pParse->db; + assert( sqlite3_mutex_held(db->mutex) ); + if( !db->init.busy ){ + rc = sqlite3Init(db, &pParse->zErrMsg); } - nameToken.z = pTrig->zName; - nameToken.n = sqlite3Strlen30(nameToken.z); - if( sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken) - && sqlite3FixTriggerStep(&sFix, pTrig->step_list) ){ - goto triggerfinish_cleanup; + if( rc!=SQLITE_OK ){ + pParse->rc = rc; + pParse->nErr++; } + return rc; +} - /* if we are not initializing, - ** build the sqlite_master entry - */ - if( !db->init.busy ){ - Vdbe *v; - char *z; - /* Make an entry in the sqlite_master table */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto triggerfinish_cleanup; - sqlite3BeginWriteOperation(pParse, 0, iDb); - z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); - sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, - pTrig->table, z); - sqlite3DbFree(db, z); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf( - db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC - ); - } +/* +** Check schema cookies in all databases. If any cookie is out +** of date set pParse->rc to SQLITE_SCHEMA. If all schema cookies +** make no changes to pParse->rc. +*/ +static void schemaIsValid(Parse *pParse){ + sqlite3 *db = pParse->db; + int iDb; + int rc; + int cookie; - if( db->init.busy ){ - Trigger *pLink = pTrig; - Hash *pHash = &db->aDb[iDb].pSchema->trigHash; - pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); - if( pTrig ){ - db->mallocFailed = 1; - }else if( pLink->pSchema==pLink->pTabSchema ){ - Table *pTab; - int n = sqlite3Strlen30(pLink->table); - pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); - assert( pTab!=0 ); - pLink->pNext = pTab->pTrigger; - pTab->pTrigger = pLink; + assert( pParse->checkSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + for(iDb=0; iDbnDb; iDb++){ + int openedTransaction = 0; /* True if a transaction is opened */ + Btree *pBt = db->aDb[iDb].pBt; /* Btree database to read cookie from */ + if( pBt==0 ) continue; + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed immediately after reading the meta-value. */ + if( !sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; + } + if( rc!=SQLITE_OK ) return; + openedTransaction = 1; } - } -triggerfinish_cleanup: - sqlite3DeleteTrigger(db, pTrig); - assert( !pParse->pNewTrigger ); - sqlite3DeleteTriggerStep(db, pStepList); -} + /* Read the schema cookie from the database. If it does not match the + ** value stored as part of the in-memory schema representation, + ** set Parse.rc to SQLITE_SCHEMA. */ + sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + sqlite3ResetInternalSchema(db, iDb); + pParse->rc = SQLITE_SCHEMA; + } -/* -** Turn a SELECT statement (that the pSelect parameter points to) into -** a trigger step. Return a pointer to a TriggerStep structure. -** -** The parser calls this routine when it finds a SELECT statement in -** body of a TRIGGER. -*/ -SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){ - TriggerStep *pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); - if( pTriggerStep==0 ) { - sqlite3SelectDelete(db, pSelect); - return 0; + /* Close the transaction, if one was opened. */ + if( openedTransaction ){ + sqlite3BtreeCommit(pBt); + } } - pTriggerStep->op = TK_SELECT; - pTriggerStep->pSelect = pSelect; - pTriggerStep->orconf = OE_Default; - return pTriggerStep; } /* -** Allocate space to hold a new trigger step. The allocated space -** holds both the TriggerStep object and the TriggerStep.target.z string. +** Convert a schema pointer into the iDb index that indicates +** which database file in db->aDb[] the schema refers to. ** -** If an OOM error occurs, NULL is returned and db->mallocFailed is set. +** If the same database is attached more than once, the first +** attached database is returned. */ -static TriggerStep *triggerStepAllocate( - sqlite3 *db, /* Database connection */ - u8 op, /* Trigger opcode */ - Token *pName /* The target name */ -){ - TriggerStep *pTriggerStep; +SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ + int i = -1000000; - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n); - if( pTriggerStep ){ - char *z = (char*)&pTriggerStep[1]; - memcpy(z, pName->z, pName->n); - pTriggerStep->target.z = z; - pTriggerStep->target.n = pName->n; - pTriggerStep->op = op; + /* If pSchema is NULL, then return -1000000. This happens when code in + ** expr.c is trying to resolve a reference to a transient table (i.e. one + ** created by a sub-select). In this case the return value of this + ** function should never be used. + ** + ** We return -1000000 instead of the more usual -1 simply because using + ** -1000000 as the incorrect index into db->aDb[] is much + ** more likely to cause a segfault than -1 (of course there are assert() + ** statements too, but it never hurts to play the odds). + */ + assert( sqlite3_mutex_held(db->mutex) ); + if( pSchema ){ + for(i=0; ALWAYS(inDb); i++){ + if( db->aDb[i].pSchema==pSchema ){ + break; + } + } + assert( i>=0 && inDb ); } - return pTriggerStep; + return i; } /* -** Build a trigger step out of an INSERT statement. Return a pointer -** to the new trigger step. -** -** The parser calls this routine when it sees an INSERT inside the -** body of a trigger. +** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ -SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( - sqlite3 *db, /* The database connection */ - Token *pTableName, /* Name of the table into which we insert */ - IdList *pColumn, /* List of columns in pTableName to insert into */ - ExprList *pEList, /* The VALUE clause: a list of values to be inserted */ - Select *pSelect, /* A SELECT statement that supplies values */ - u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ ){ - TriggerStep *pTriggerStep; - - assert(pEList == 0 || pSelect == 0); - assert(pEList != 0 || pSelect != 0 || db->mallocFailed); + Parse *pParse; /* Parsing context */ + char *zErrMsg = 0; /* Error message */ + int rc = SQLITE_OK; /* Result code */ + int i; /* Loop counter */ - pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); - if( pTriggerStep ){ - pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - pTriggerStep->pIdList = pColumn; - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); - pTriggerStep->orconf = orconf; - }else{ - sqlite3IdListDelete(db, pColumn); + /* Allocate the parsing context */ + pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); + if( pParse==0 ){ + rc = SQLITE_NOMEM; + goto end_prepare; } - sqlite3ExprListDelete(db, pEList); - sqlite3SelectDelete(db, pSelect); - - return pTriggerStep; -} + pParse->pReprepare = pReprepare; + assert( ppStmt && *ppStmt==0 ); + assert( !db->mallocFailed ); + assert( sqlite3_mutex_held(db->mutex) ); -/* -** Construct a trigger step that implements an UPDATE statement and return -** a pointer to that trigger step. The parser calls this routine when it -** sees an UPDATE statement inside the body of a CREATE TRIGGER. -*/ -SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( - sqlite3 *db, /* The database connection */ - Token *pTableName, /* Name of the table to be updated */ - ExprList *pEList, /* The SET clause: list of column and new values */ - Expr *pWhere, /* The WHERE clause */ - u8 orconf /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ -){ - TriggerStep *pTriggerStep; - - pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName); - if( pTriggerStep ){ - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pTriggerStep->orconf = orconf; + /* Check to verify that it is possible to get a read lock on all + ** database schemas. The inability to get a read lock indicates that + ** some other database connection is holding a write-lock, which in + ** turn means that the other connection has made uncommitted changes + ** to the schema. + ** + ** Were we to proceed and prepare the statement against the uncommitted + ** schema changes and if those schema changes are subsequently rolled + ** back and different changes are made in their place, then when this + ** prepared statement goes to run the schema cookie would fail to detect + ** the schema change. Disaster would follow. + ** + ** This thread is currently holding mutexes on all Btrees (because + ** of the sqlite3BtreeEnterAll() in sqlite3LockAndPrepare()) so it + ** is not possible for another thread to start a new schema change + ** while this routine is running. Hence, we do not need to hold + ** locks on the schema, we just need to make sure nobody else is + ** holding them. + ** + ** Note that setting READ_UNCOMMITTED overrides most lock detection, + ** but it does *not* override schema lock detection, so this all still + ** works even if READ_UNCOMMITTED is set. + */ + for(i=0; inDb; i++) { + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + assert( sqlite3BtreeHoldsMutex(pBt) ); + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zName; + sqlite3Error(db, rc, "database schema is locked: %s", zDb); + testcase( db->flags & SQLITE_ReadUncommitted ); + goto end_prepare; + } + } } - sqlite3ExprListDelete(db, pEList); - sqlite3ExprDelete(db, pWhere); - return pTriggerStep; -} -/* -** Construct a trigger step that implements a DELETE statement and return -** a pointer to that trigger step. The parser calls this routine when it -** sees a DELETE statement inside the body of a CREATE TRIGGER. -*/ -SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( - sqlite3 *db, /* Database connection */ - Token *pTableName, /* The table from which rows are deleted */ - Expr *pWhere /* The WHERE clause */ -){ - TriggerStep *pTriggerStep; + sqlite3VtabUnlockList(db); - pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName); - if( pTriggerStep ){ - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pTriggerStep->orconf = OE_Default; + pParse->db = db; + pParse->nQueryLoop = (double)1; + if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ + char *zSqlCopy; + int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + testcase( nBytes==mxLen ); + testcase( nBytes==mxLen+1 ); + if( nBytes>mxLen ){ + sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + rc = sqlite3ApiExit(db, SQLITE_TOOBIG); + goto end_prepare; + } + zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); + if( zSqlCopy ){ + sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); + sqlite3DbFree(db, zSqlCopy); + pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; + }else{ + pParse->zTail = &zSql[nBytes]; + } + }else{ + sqlite3RunParser(pParse, zSql, &zErrMsg); } - sqlite3ExprDelete(db, pWhere); - return pTriggerStep; -} + assert( 1==(int)pParse->nQueryLoop ); -/* -** Recursively delete a Trigger structure -*/ -SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3 *db, Trigger *pTrigger){ - if( pTrigger==0 ) return; - sqlite3DeleteTriggerStep(db, pTrigger->step_list); - sqlite3DbFree(db, pTrigger->zName); - sqlite3DbFree(db, pTrigger->table); - sqlite3ExprDelete(db, pTrigger->pWhen); - sqlite3IdListDelete(db, pTrigger->pColumns); - sqlite3DbFree(db, pTrigger); -} + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; + if( pParse->checkSchema ){ + schemaIsValid(pParse); + } + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pzTail ){ + *pzTail = pParse->zTail; + } + rc = pParse->rc; -/* -** This function is called to drop a trigger from the database schema. -** -** This may be called directly from the parser and therefore identifies -** the trigger by name. The sqlite3DropTriggerPtr() routine does the -** same job as this routine except it takes a pointer to the trigger -** instead of the trigger name. -**/ -SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ - Trigger *pTrigger = 0; - int i; - const char *zDb; - const char *zName; - int nName; - sqlite3 *db = pParse->db; +#ifndef SQLITE_OMIT_EXPLAIN + if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ + static const char * const azColName[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", + "selectid", "order", "from", "detail" + }; + int iFirst, mx; + if( pParse->explain==2 ){ + sqlite3VdbeSetNumCols(pParse->pVdbe, 4); + iFirst = 8; + mx = 12; + }else{ + sqlite3VdbeSetNumCols(pParse->pVdbe, 8); + iFirst = 0; + mx = 8; + } + for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, + azColName[i], SQLITE_STATIC); + } + } +#endif - if( db->mallocFailed ) goto drop_trigger_cleanup; - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ - goto drop_trigger_cleanup; + assert( db->init.busy==0 || saveSqlFlag==0 ); + if( db->init.busy==0 ){ + Vdbe *pVdbe = pParse->pVdbe; + sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); + } + if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ + sqlite3VdbeFinalize(pParse->pVdbe); + assert(!(*ppStmt)); + }else{ + *ppStmt = (sqlite3_stmt*)pParse->pVdbe; } - assert( pName->nSrc==1 ); - zDb = pName->a[0].zDatabase; - zName = pName->a[0].zName; - nName = sqlite3Strlen30(zName); - for(i=OMIT_TEMPDB; inDb; i++){ - int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; - pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); - if( pTrigger ) break; + if( zErrMsg ){ + sqlite3Error(db, rc, "%s", zErrMsg); + sqlite3DbFree(db, zErrMsg); + }else{ + sqlite3Error(db, rc, 0); } - if( !pTrigger ){ - if( !noErr ){ - sqlite3ErrorMsg(pParse, "no such trigger: %S", pName, 0); - } - pParse->checkSchema = 1; - goto drop_trigger_cleanup; + + /* Delete any TriggerPrg structures allocated while parsing this statement. */ + while( pParse->pTriggerPrg ){ + TriggerPrg *pT = pParse->pTriggerPrg; + pParse->pTriggerPrg = pT->pNext; + sqlite3DbFree(db, pT); } - sqlite3DropTriggerPtr(pParse, pTrigger); -drop_trigger_cleanup: - sqlite3SrcListDelete(db, pName); -} +end_prepare: -/* -** Return a pointer to the Table structure for the table that a trigger -** is set on. -*/ -static Table *tableOfTrigger(Trigger *pTrigger){ - int n = sqlite3Strlen30(pTrigger->table); - return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); + sqlite3StackFree(db, pParse); + rc = sqlite3ApiExit(db, rc); + assert( (rc&db->errMask)==rc ); + return rc; +} +static int sqlite3LockAndPrepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + Vdbe *pOld, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + assert( ppStmt!=0 ); + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); + if( rc==SQLITE_SCHEMA ){ + sqlite3_finalize(*ppStmt); + rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return rc; } - /* -** Drop a trigger given a pointer to that trigger. +** Rerun the compilation of a statement after a schema change. +** +** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, +** if the statement cannot be recompiled because another connection has +** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error +** occurs, return SQLITE_SCHEMA. */ -SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ - Table *pTable; - Vdbe *v; - sqlite3 *db = pParse->db; - int iDb; - - iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); - assert( iDb>=0 && iDbnDb ); - pTable = tableOfTrigger(pTrigger); - assert( pTable ); - assert( pTable->pSchema==pTrigger->pSchema || iDb==1 ); -#ifndef SQLITE_OMIT_AUTHORIZATION - { - int code = SQLITE_DROP_TRIGGER; - const char *zDb = db->aDb[iDb].zName; - const char *zTab = SCHEMA_TABLE(iDb); - if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; - if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || - sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ - return; - } - } -#endif - - /* Generate code to destroy the database record of the trigger. - */ - assert( pTable!=0 ); - if( (v = sqlite3GetVdbe(pParse))!=0 ){ - int base; - static const VdbeOpList dropTrigger[] = { - { OP_Rewind, 0, ADDR(9), 0}, - { OP_String8, 0, 1, 0}, /* 1 */ - { OP_Column, 0, 1, 2}, - { OP_Ne, 2, ADDR(8), 1}, - { OP_String8, 0, 1, 0}, /* 4: "trigger" */ - { OP_Column, 0, 0, 2}, - { OP_Ne, 2, ADDR(8), 1}, - { OP_Delete, 0, 0, 0}, - { OP_Next, 0, ADDR(1), 0}, /* 8 */ - }; +SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ + int rc; + sqlite3_stmt *pNew; + const char *zSql; + sqlite3 *db; - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3OpenMasterTable(pParse, iDb); - base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger); - sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, 0); - sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_Close, 0, 0); - sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); - if( pParse->nMem<3 ){ - pParse->nMem = 3; + assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); + zSql = sqlite3_sql((sqlite3_stmt *)p); + assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ + db = sqlite3VdbeDb(p); + assert( sqlite3_mutex_held(db->mutex) ); + rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); + if( rc ){ + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; } + assert( pNew==0 ); + return rc; + }else{ + assert( pNew!=0 ); } + sqlite3VdbeSwap((Vdbe*)pNew, p); + sqlite3TransferBindings(pNew, (sqlite3_stmt*)p); + sqlite3VdbeResetStepResult((Vdbe*)pNew); + sqlite3VdbeFinalize((Vdbe*)pNew); + return SQLITE_OK; } + /* -** Remove a trigger from the hash tables of the sqlite* pointer. +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. */ -SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){ - Hash *pHash = &(db->aDb[iDb].pSchema->trigHash); - Trigger *pTrigger; - pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); - if( ALWAYS(pTrigger) ){ - if( pTrigger->pSchema==pTrigger->pTabSchema ){ - Table *pTab = tableOfTrigger(pTrigger); - Trigger **pp; - for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); - *pp = (*pp)->pNext; - } - sqlite3DeleteTrigger(db, pTrigger); - db->flags |= SQLITE_InternChanges; - } +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,1,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; } + +#ifndef SQLITE_OMIT_UTF16 /* -** pEList is the SET clause of an UPDATE statement. Each entry -** in pEList is of the format =. If any of the entries -** in pEList have an which matches an identifier in pIdList, -** then return TRUE. If pIdList==NULL, then it is considered a -** wildcard that matches anything. Likewise if pEList==NULL then -** it matches anything so always return true. Return false only -** if there is no match. +** Compile the UTF-16 encoded SQL statement zSql into a statement handle. */ -static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ - int e; - if( pIdList==0 || NEVER(pEList==0) ) return 1; - for(e=0; enExpr; e++){ - if( sqlite3IdListIndex(pIdList, pEList->a[e].zName)>=0 ) return 1; +static int sqlite3Prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to save SQL text into the sqlite3_stmt */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + /* This function currently works by first transforming the UTF-16 + ** encoded string to UTF-8, then invoking sqlite3_prepare(). The + ** tricky bit is figuring out the pointer to return in *pzTail. + */ + char *zSql8; + const char *zTail8 = 0; + int rc = SQLITE_OK; + + assert( ppStmt ); + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; } - return 0; + sqlite3_mutex_enter(db->mutex); + zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); + if( zSql8 ){ + rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, 0, ppStmt, &zTail8); + } + + if( zTail8 && pzTail ){ + /* If sqlite3_prepare returns a tail pointer, we calculate the + ** equivalent pointer into the UTF-16 string by counting the unicode + ** characters between zSql8 and zTail8, and then returning a pointer + ** the same number of characters into the UTF-16 string. + */ + int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + } + sqlite3DbFree(db, zSql8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* -** Return a list of all triggers on table pTab if there exists at least -** one trigger that must be fired when an operation of type 'op' is -** performed on the table, and, if that operation is an UPDATE, if at -** least one of the columns in pChanges is being modified. +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. */ -SQLITE_PRIVATE Trigger *sqlite3TriggersExist( - Parse *pParse, /* Parse context */ - Table *pTab, /* The table the contains the triggers */ - int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ - ExprList *pChanges, /* Columns that change in an UPDATE statement */ - int *pMask /* OUT: Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ ){ - int mask = 0; - Trigger *pList = sqlite3TriggerList(pParse, pTab); - Trigger *p; - assert( pList==0 || IsVirtual(pTab)==0 ); - for(p=pList; p; p=p->pNext){ - if( p->op==op && checkColumnOverlap(p->pColumns, pChanges) ){ - mask |= p->tr_tm; - } - } - if( pMask ){ - *pMask = mask; - } - return (mask ? pList : 0); + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,1,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; } +#endif /* SQLITE_OMIT_UTF16 */ + +/************** End of prepare.c *********************************************/ +/************** Begin file select.c ******************************************/ /* -** Convert the pStep->target token into a SrcList and return a pointer -** to that SrcList. +** 2001 September 15 ** -** This routine adds a specific database name, if needed, to the target when -** forming the SrcList. This prevents a trigger in one database from -** referring to a target in another database. An exception is when the -** trigger is in TEMP in which case it can refer to any other database it -** wants. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle SELECT statements in SQLite. */ -static SrcList *targetSrcList( - Parse *pParse, /* The parsing context */ - TriggerStep *pStep /* The trigger containing the target token */ -){ - int iDb; /* Index of the database to use */ - SrcList *pSrc; /* SrcList to be returned */ - pSrc = sqlite3SrcListAppend(pParse->db, 0, &pStep->target, 0); - if( pSrc ){ - assert( pSrc->nSrc>0 ); - assert( pSrc->a!=0 ); - iDb = sqlite3SchemaToIndex(pParse->db, pStep->pTrig->pSchema); - if( iDb==0 || iDb>=2 ){ - sqlite3 *db = pParse->db; - assert( iDbdb->nDb ); - pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); - } - } - return pSrc; -} /* -** Generate VDBE code for the statements inside the body of a single -** trigger. +** Delete all the content of a Select structure but do not deallocate +** the select structure itself. */ -static int codeTriggerProgram( - Parse *pParse, /* The parser context */ - TriggerStep *pStepList, /* List of statements inside the trigger body */ - int orconf /* Conflict algorithm. (OE_Abort, etc) */ -){ - TriggerStep *pStep; - Vdbe *v = pParse->pVdbe; - sqlite3 *db = pParse->db; - - assert( pParse->pTriggerTab && pParse->pToplevel ); - assert( pStepList ); - assert( v!=0 ); - for(pStep=pStepList; pStep; pStep=pStep->pNext){ - /* Figure out the ON CONFLICT policy that will be used for this step - ** of the trigger program. If the statement that caused this trigger - ** to fire had an explicit ON CONFLICT, then use it. Otherwise, use - ** the ON CONFLICT policy that was specified as part of the trigger - ** step statement. Example: - ** - ** CREATE TRIGGER AFTER INSERT ON t1 BEGIN; - ** INSERT OR REPLACE INTO t2 VALUES(new.a, new.b); - ** END; - ** - ** INSERT INTO t1 ... ; -- insert into t2 uses REPLACE policy - ** INSERT OR IGNORE INTO t1 ... ; -- insert into t2 uses IGNORE policy - */ - pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; - - switch( pStep->op ){ - case TK_UPDATE: { - sqlite3Update(pParse, - targetSrcList(pParse, pStep), - sqlite3ExprListDup(db, pStep->pExprList, 0), - sqlite3ExprDup(db, pStep->pWhere, 0), - pParse->eOrconf - ); - break; - } - case TK_INSERT: { - sqlite3Insert(pParse, - targetSrcList(pParse, pStep), - sqlite3ExprListDup(db, pStep->pExprList, 0), - sqlite3SelectDup(db, pStep->pSelect, 0), - sqlite3IdListDup(db, pStep->pIdList), - pParse->eOrconf - ); - break; - } - case TK_DELETE: { - sqlite3DeleteFrom(pParse, - targetSrcList(pParse, pStep), - sqlite3ExprDup(db, pStep->pWhere, 0) - ); - break; - } - default: assert( pStep->op==TK_SELECT ); { - SelectDest sDest; - Select *pSelect = sqlite3SelectDup(db, pStep->pSelect, 0); - sqlite3SelectDestInit(&sDest, SRT_Discard, 0); - sqlite3Select(pParse, pSelect, &sDest); - sqlite3SelectDelete(db, pSelect); - break; - } - } - if( pStep->op!=TK_SELECT ){ - sqlite3VdbeAddOp0(v, OP_ResetCount); - } - } +static void clearSelect(sqlite3 *db, Select *p){ + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3SelectDelete(db, p->pPrior); + sqlite3ExprDelete(db, p->pLimit); + sqlite3ExprDelete(db, p->pOffset); +} - return 0; +/* +** Initialize a SelectDest structure. +*/ +SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ + pDest->eDest = (u8)eDest; + pDest->iParm = iParm; + pDest->affinity = 0; + pDest->iMem = 0; + pDest->nMem = 0; } -#ifdef SQLITE_DEBUG + /* -** This function is used to add VdbeComment() annotations to a VDBE -** program. It is not used in production code, only for debugging. +** Allocate a new Select structure and return a pointer to that +** structure. */ -static const char *onErrorText(int onError){ - switch( onError ){ - case OE_Abort: return "abort"; - case OE_Rollback: return "rollback"; - case OE_Fail: return "fail"; - case OE_Replace: return "replace"; - case OE_Ignore: return "ignore"; - case OE_Default: return "default"; +SQLITE_PRIVATE Select *sqlite3SelectNew( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* which columns to include in the result */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* the WHERE clause */ + ExprList *pGroupBy, /* the GROUP BY clause */ + Expr *pHaving, /* the HAVING clause */ + ExprList *pOrderBy, /* the ORDER BY clause */ + int isDistinct, /* true if the DISTINCT keyword is present */ + Expr *pLimit, /* LIMIT value. NULL means not used */ + Expr *pOffset /* OFFSET value. NULL means no offset */ +){ + Select *pNew; + Select standin; + sqlite3 *db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ + if( pNew==0 ){ + pNew = &standin; + memset(pNew, 0, sizeof(*pNew)); } - return "n/a"; + if( pEList==0 ){ + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0)); + } + pNew->pEList = pEList; + pNew->pSrc = pSrc; + pNew->pWhere = pWhere; + pNew->pGroupBy = pGroupBy; + pNew->pHaving = pHaving; + pNew->pOrderBy = pOrderBy; + pNew->selFlags = isDistinct ? SF_Distinct : 0; + pNew->op = TK_SELECT; + pNew->pLimit = pLimit; + pNew->pOffset = pOffset; + assert( pOffset==0 || pLimit!=0 ); + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->addrOpenEphm[2] = -1; + if( db->mallocFailed ) { + clearSelect(db, pNew); + if( pNew!=&standin ) sqlite3DbFree(db, pNew); + pNew = 0; + } + return pNew; } -#endif /* -** Parse context structure pFrom has just been used to create a sub-vdbe -** (trigger program). If an error has occurred, transfer error information -** from pFrom to pTo. +** Delete the given Select structure and all of its substructures. */ -static void transferParseError(Parse *pTo, Parse *pFrom){ - assert( pFrom->zErrMsg==0 || pFrom->nErr ); - assert( pTo->zErrMsg==0 || pTo->nErr ); - if( pTo->nErr==0 ){ - pTo->zErrMsg = pFrom->zErrMsg; - pTo->nErr = pFrom->nErr; - }else{ - sqlite3DbFree(pFrom->db, pFrom->zErrMsg); +SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ + if( p ){ + clearSelect(db, p); + sqlite3DbFree(db, p); } } /* -** Create and populate a new TriggerPrg object with a sub-program -** implementing trigger pTrigger with ON CONFLICT policy orconf. +** Given 1 to 3 identifiers preceeding the JOIN keyword, determine the +** type of join. Return an integer constant that expresses that type +** in terms of the following bit values: +** +** JT_INNER +** JT_CROSS +** JT_OUTER +** JT_NATURAL +** JT_LEFT +** JT_RIGHT +** +** A full outer join is the combination of JT_LEFT and JT_RIGHT. +** +** If an illegal or unsupported join type is seen, then still return +** a join type, but put an error in the pParse structure. */ -static TriggerPrg *codeRowTrigger( - Parse *pParse, /* Current parse context */ - Trigger *pTrigger, /* Trigger to code */ - Table *pTab, /* The table pTrigger is attached to */ - int orconf /* ON CONFLICT policy to code trigger program with */ -){ - Parse *pTop = sqlite3ParseToplevel(pParse); - sqlite3 *db = pParse->db; /* Database handle */ - TriggerPrg *pPrg; /* Value to return */ - Expr *pWhen = 0; /* Duplicate of trigger WHEN expression */ - Vdbe *v; /* Temporary VM */ - NameContext sNC; /* Name context for sub-vdbe */ - SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ - Parse *pSubParse; /* Parse context for sub-vdbe */ - int iEndTrigger = 0; /* Label to jump to if WHEN is false */ - - assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); - assert( pTop->pVdbe ); - - /* Allocate the TriggerPrg and SubProgram objects. To ensure that they - ** are freed if an error occurs, link them into the Parse.pTriggerPrg - ** list of the top-level Parse object sooner rather than later. */ - pPrg = sqlite3DbMallocZero(db, sizeof(TriggerPrg)); - if( !pPrg ) return 0; - pPrg->pNext = pTop->pTriggerPrg; - pTop->pTriggerPrg = pPrg; - pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); - if( !pProgram ) return 0; - sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); - pPrg->pTrigger = pTrigger; - pPrg->orconf = orconf; - pPrg->aColmask[0] = 0xffffffff; - pPrg->aColmask[1] = 0xffffffff; - - /* Allocate and populate a new Parse context to use for coding the - ** trigger sub-program. */ - pSubParse = sqlite3StackAllocZero(db, sizeof(Parse)); - if( !pSubParse ) return 0; - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pSubParse; - pSubParse->db = db; - pSubParse->pTriggerTab = pTab; - pSubParse->pToplevel = pTop; - pSubParse->zAuthContext = pTrigger->zName; - pSubParse->eTriggerOp = pTrigger->op; - pSubParse->nQueryLoop = pParse->nQueryLoop; - - v = sqlite3GetVdbe(pSubParse); - if( v ){ - VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", - pTrigger->zName, onErrorText(orconf), - (pTrigger->tr_tm==TRIGGER_BEFORE ? "BEFORE" : "AFTER"), - (pTrigger->op==TK_UPDATE ? "UPDATE" : ""), - (pTrigger->op==TK_INSERT ? "INSERT" : ""), - (pTrigger->op==TK_DELETE ? "DELETE" : ""), - pTab->zName - )); -#ifndef SQLITE_OMIT_TRACE - sqlite3VdbeChangeP4(v, -1, - sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC - ); -#endif - - /* If one was specified, code the WHEN clause. If it evaluates to false - ** (or NULL) the sub-vdbe is immediately halted by jumping to the - ** OP_Halt inserted at the end of the program. */ - if( pTrigger->pWhen ){ - pWhen = sqlite3ExprDup(db, pTrigger->pWhen, 0); - if( SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) - && db->mallocFailed==0 - ){ - iEndTrigger = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); +SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ + int jointype = 0; + Token *apAll[3]; + Token *p; + /* 0123456789 123456789 123456789 123 */ + static const char zKeyText[] = "naturaleftouterightfullinnercross"; + static const struct { + u8 i; /* Beginning of keyword text in zKeyText[] */ + u8 nChar; /* Length of the keyword in characters */ + u8 code; /* Join type mask */ + } aKeyword[] = { + /* natural */ { 0, 7, JT_NATURAL }, + /* left */ { 6, 4, JT_LEFT|JT_OUTER }, + /* outer */ { 10, 5, JT_OUTER }, + /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, + /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, + /* inner */ { 23, 5, JT_INNER }, + /* cross */ { 28, 5, JT_INNER|JT_CROSS }, + }; + int i, j; + apAll[0] = pA; + apAll[1] = pB; + apAll[2] = pC; + for(i=0; i<3 && apAll[i]; i++){ + p = apAll[i]; + for(j=0; jn==aKeyword[j].nChar + && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ + jointype |= aKeyword[j].code; + break; } - sqlite3ExprDelete(db, pWhen); - } - - /* Code the trigger program into the sub-vdbe. */ - codeTriggerProgram(pSubParse, pTrigger->step_list, orconf); - - /* Insert an OP_Halt at the end of the sub-program. */ - if( iEndTrigger ){ - sqlite3VdbeResolveLabel(v, iEndTrigger); } - sqlite3VdbeAddOp0(v, OP_Halt); - VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); - - transferParseError(pParse, pSubParse); - if( db->mallocFailed==0 ){ - pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); + testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); + if( j>=ArraySize(aKeyword) ){ + jointype |= JT_ERROR; + break; } - pProgram->nMem = pSubParse->nMem; - pProgram->nCsr = pSubParse->nTab; - pProgram->token = (void *)pTrigger; - pPrg->aColmask[0] = pSubParse->oldmask; - pPrg->aColmask[1] = pSubParse->newmask; - sqlite3VdbeDelete(v); } + if( + (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || + (jointype & JT_ERROR)!=0 + ){ + const char *zSp = " "; + assert( pB!=0 ); + if( pC==0 ){ zSp++; } + sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " + "%T %T%s%T", pA, pB, zSp, pC); + jointype = JT_INNER; + }else if( (jointype & JT_OUTER)!=0 + && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ + sqlite3ErrorMsg(pParse, + "RIGHT and FULL OUTER JOINs are not currently supported"); + jointype = JT_INNER; + } + return jointype; +} - assert( !pSubParse->pAinc && !pSubParse->pZombieTab ); - assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); - sqlite3StackFree(db, pSubParse); - - return pPrg; +/* +** Return the index of a column in a table. Return -1 if the column +** is not contained in the table. +*/ +static int columnIndex(Table *pTab, const char *zCol){ + int i; + for(i=0; inCol; i++){ + if( sqlite3StrICmp(pTab->aCol[i].zName, zCol)==0 ) return i; + } + return -1; } - + /* -** Return a pointer to a TriggerPrg object containing the sub-program for -** trigger pTrigger with default ON CONFLICT algorithm orconf. If no such -** TriggerPrg object exists, a new object is allocated and populated before -** being returned. +** Search the first N tables in pSrc, from left to right, looking for a +** table that has a column named zCol. +** +** When found, set *piTab and *piCol to the table index and column index +** of the matching column and return TRUE. +** +** If not found, return FALSE. */ -static TriggerPrg *getRowTrigger( - Parse *pParse, /* Current parse context */ - Trigger *pTrigger, /* Trigger to code */ - Table *pTab, /* The table trigger pTrigger is attached to */ - int orconf /* ON CONFLICT algorithm. */ +static int tableAndColumnIndex( + SrcList *pSrc, /* Array of tables to search */ + int N, /* Number of tables in pSrc->a[] to search */ + const char *zCol, /* Name of the column we are looking for */ + int *piTab, /* Write index of pSrc->a[] here */ + int *piCol /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ ){ - Parse *pRoot = sqlite3ParseToplevel(pParse); - TriggerPrg *pPrg; - - assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); - - /* It may be that this trigger has already been coded (or is in the - ** process of being coded). If this is the case, then an entry with - ** a matching TriggerPrg.pTrigger field will be present somewhere - ** in the Parse.pTriggerPrg list. Search for such an entry. */ - for(pPrg=pRoot->pTriggerPrg; - pPrg && (pPrg->pTrigger!=pTrigger || pPrg->orconf!=orconf); - pPrg=pPrg->pNext - ); + int i; /* For looping over tables in pSrc */ + int iCol; /* Index of column matching zCol */ - /* If an existing TriggerPrg could not be located, create a new one. */ - if( !pPrg ){ - pPrg = codeRowTrigger(pParse, pTrigger, pTab, orconf); + assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ + for(i=0; ia[i].pTab, zCol); + if( iCol>=0 ){ + if( piTab ){ + *piTab = i; + *piCol = iCol; + } + return 1; + } } - - return pPrg; + return 0; } /* -** Generate code for the trigger program associated with trigger p on -** table pTab. The reg, orconf and ignoreJump parameters passed to this -** function are the same as those described in the header function for -** sqlite3CodeRowTrigger() +** This function is used to add terms implied by JOIN syntax to the +** WHERE clause expression of a SELECT statement. The new term, which +** is ANDed with the existing WHERE clause, is of the form: +** +** (tab1.col1 = tab2.col2) +** +** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the +** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is +** column iColRight of tab2. */ -SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( - Parse *pParse, /* Parse context */ - Trigger *p, /* Trigger to code */ - Table *pTab, /* The table to code triggers from */ - int reg, /* Reg array containing OLD.* and NEW.* values */ - int orconf, /* ON CONFLICT policy */ - int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +static void addWhereTerm( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* List of tables in FROM clause */ + int iLeft, /* Index of first table to join in pSrc */ + int iColLeft, /* Index of column in first table */ + int iRight, /* Index of second table in pSrc */ + int iColRight, /* Index of column in second table */ + int isOuterJoin, /* True if this is an OUTER join */ + Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ ){ - Vdbe *v = sqlite3GetVdbe(pParse); /* Main VM */ - TriggerPrg *pPrg; - pPrg = getRowTrigger(pParse, p, pTab, orconf); - assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); + sqlite3 *db = pParse->db; + Expr *pE1; + Expr *pE2; + Expr *pEq; - /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program - ** is a pointer to the sub-vdbe containing the trigger program. */ - if( pPrg ){ - int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); + assert( iLeftnSrc>iRight ); + assert( pSrc->a[iLeft].pTab ); + assert( pSrc->a[iRight].pTab ); - sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); - VdbeComment( - (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); + pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); - /* Set the P5 operand of the OP_Program instruction to non-zero if - ** recursive invocation of this trigger program is disallowed. Recursive - ** invocation is disallowed if (a) the sub-program is really a trigger, - ** not a foreign key action, and (b) the flag to enable recursive triggers - ** is clear. */ - sqlite3VdbeChangeP5(v, (u8)bRecursive); + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); + if( pEq && isOuterJoin ){ + ExprSetProperty(pEq, EP_FromJoin); + assert( !ExprHasAnyProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetIrreducible(pEq); + pEq->iRightJoinTable = (i16)pE2->iTable; } + *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); } /* -** This is called to code the required FOR EACH ROW triggers for an operation -** on table pTab. The operation to code triggers for (INSERT, UPDATE or DELETE) -** is given by the op paramater. The tr_tm parameter determines whether the -** BEFORE or AFTER triggers are coded. If the operation is an UPDATE, then -** parameter pChanges is passed the list of columns being modified. -** -** If there are no triggers that fire at the specified time for the specified -** operation on pTab, this function is a no-op. +** Set the EP_FromJoin property on all terms of the given expression. +** And set the Expr.iRightJoinTable to iTable for every term in the +** expression. ** -** The reg argument is the address of the first in an array of registers -** that contain the values substituted for the new.* and old.* references -** in the trigger program. If N is the number of columns in table pTab -** (a copy of pTab->nCol), then registers are populated as follows: +** The EP_FromJoin property is used on terms of an expression to tell +** the LEFT OUTER JOIN processing logic that this term is part of the +** join restriction specified in the ON or USING clause and not a part +** of the more general WHERE clause. These terms are moved over to the +** WHERE clause during join processing but we need to remember that they +** originated in the ON or USING clause. ** -** Register Contains -** ------------------------------------------------------ -** reg+0 OLD.rowid -** reg+1 OLD.* value of left-most column of pTab -** ... ... -** reg+N OLD.* value of right-most column of pTab -** reg+N+1 NEW.rowid -** reg+N+2 OLD.* value of left-most column of pTab -** ... ... -** reg+N+N+1 NEW.* value of right-most column of pTab +** The Expr.iRightJoinTable tells the WHERE clause processing that the +** expression depends on table iRightJoinTable even if that table is not +** explicitly mentioned in the expression. That information is needed +** for cases like this: ** -** For ON DELETE triggers, the registers containing the NEW.* values will -** never be accessed by the trigger program, so they are not allocated or -** populated by the caller (there is no data to populate them with anyway). -** Similarly, for ON INSERT triggers the values stored in the OLD.* registers -** are never accessed, and so are not allocated by the caller. So, for an -** ON INSERT trigger, the value passed to this function as parameter reg -** is not a readable register, although registers (reg+N) through -** (reg+N+N+1) are. +** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5 ** -** Parameter orconf is the default conflict resolution algorithm for the -** trigger program to use (REPLACE, IGNORE etc.). Parameter ignoreJump -** is the instruction that control should jump to if a trigger program -** raises an IGNORE exception. +** The where clause needs to defer the handling of the t1.x=5 +** term until after the t2 loop of the join. In that way, a +** NULL t2 row will be inserted whenever t1.x!=5. If we do not +** defer the handling of t1.x=5, it will be processed immediately +** after the t1 loop and rows with t1.x!=5 will never appear in +** the output, which is incorrect. */ -SQLITE_PRIVATE void sqlite3CodeRowTrigger( - Parse *pParse, /* Parse context */ - Trigger *pTrigger, /* List of triggers on table pTab */ - int op, /* One of TK_UPDATE, TK_INSERT, TK_DELETE */ - ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ - int tr_tm, /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ - Table *pTab, /* The table to code triggers from */ - int reg, /* The first in an array of registers (see above) */ - int orconf, /* ON CONFLICT policy */ - int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ -){ - Trigger *p; /* Used to iterate through pTrigger list */ +static void setJoinExpr(Expr *p, int iTable){ + while( p ){ + ExprSetProperty(p, EP_FromJoin); + assert( !ExprHasAnyProperty(p, EP_TokenOnly|EP_Reduced) ); + ExprSetIrreducible(p); + p->iRightJoinTable = (i16)iTable; + setJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} - assert( op==TK_UPDATE || op==TK_INSERT || op==TK_DELETE ); - assert( tr_tm==TRIGGER_BEFORE || tr_tm==TRIGGER_AFTER ); - assert( (op==TK_UPDATE)==(pChanges!=0) ); +/* +** This routine processes the join information for a SELECT statement. +** ON and USING clauses are converted into extra terms of the WHERE clause. +** NATURAL joins also create extra WHERE clause terms. +** +** The terms of a FROM clause are contained in the Select.pSrc structure. +** The left most table is the first entry in Select.pSrc. The right-most +** table is the last entry. The join operator is held in the entry to +** the left. Thus entry 0 contains the join operator for the join between +** entries 0 and 1. Any ON or USING clauses associated with the join are +** also attached to the left entry. +** +** This routine returns the number of errors encountered. +*/ +static int sqliteProcessJoin(Parse *pParse, Select *p){ + SrcList *pSrc; /* All tables in the FROM clause */ + int i, j; /* Loop counters */ + struct SrcList_item *pLeft; /* Left table being joined */ + struct SrcList_item *pRight; /* Right table being joined */ - for(p=pTrigger; p; p=p->pNext){ + pSrc = p->pSrc; + pLeft = &pSrc->a[0]; + pRight = &pLeft[1]; + for(i=0; inSrc-1; i++, pRight++, pLeft++){ + Table *pLeftTab = pLeft->pTab; + Table *pRightTab = pRight->pTab; + int isOuter; - /* Sanity checking: The schema for the trigger and for the table are - ** always defined. The trigger must be in the same schema as the table - ** or else it must be a TEMP trigger. */ - assert( p->pSchema!=0 ); - assert( p->pTabSchema!=0 ); - assert( p->pSchema==p->pTabSchema - || p->pSchema==pParse->db->aDb[1].pSchema ); + if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; + isOuter = (pRight->jointype & JT_OUTER)!=0; - /* Determine whether we should code this trigger */ - if( p->op==op - && p->tr_tm==tr_tm - && checkColumnOverlap(p->pColumns, pChanges) - ){ - sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); + /* When the NATURAL keyword is present, add WHERE clause terms for + ** every column that the two tables have in common. + */ + if( pRight->jointype & JT_NATURAL ){ + if( pRight->pOn || pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "a NATURAL join may not have " + "an ON or USING clause", 0); + return 1; + } + for(j=0; jnCol; j++){ + char *zName; /* Name of column in the right table */ + int iLeft; /* Matching left table */ + int iLeftCol; /* Matching column in the left table */ + + zName = pRightTab->aCol[j].zName; + if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) ){ + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, + isOuter, &p->pWhere); + } + } + } + + /* Disallow both ON and USING clauses in the same join + */ + if( pRight->pOn && pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "cannot have both ON and USING " + "clauses in the same join"); + return 1; + } + + /* Add the ON clause to the end of the WHERE clause, connected by + ** an AND operator. + */ + if( pRight->pOn ){ + if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); + p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); + pRight->pOn = 0; + } + + /* Create extra terms on the WHERE clause for each column named + ** in the USING clause. Example: If the two tables to be joined are + ** A and B and the USING clause names X, Y, and Z, then add this + ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z + ** Report an error if any column mentioned in the USING clause is + ** not contained in both tables to be joined. + */ + if( pRight->pUsing ){ + IdList *pList = pRight->pUsing; + for(j=0; jnId; j++){ + char *zName; /* Name of the term in the USING clause */ + int iLeft; /* Table on the left with matching column name */ + int iLeftCol; /* Column number of matching column on the left */ + int iRightCol; /* Column number of matching column on the right */ + + zName = pList->a[j].zName; + iRightCol = columnIndex(pRightTab, zName); + if( iRightCol<0 + || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) + ){ + sqlite3ErrorMsg(pParse, "cannot join using column %s - column " + "not present in both tables", zName); + return 1; + } + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, + isOuter, &p->pWhere); + } } } + return 0; } /* -** Triggers may access values stored in the old.* or new.* pseudo-table. -** This function returns a 32-bit bitmask indicating which columns of the -** old.* or new.* tables actually are used by triggers. This information -** may be used by the caller, for example, to avoid having to load the entire -** old.* record into memory when executing an UPDATE or DELETE command. -** -** Bit 0 of the returned mask is set if the left-most column of the -** table may be accessed using an [old|new].reference. Bit 1 is set if -** the second leftmost column value is required, and so on. If there -** are more than 32 columns in the table, and at least one of the columns -** with an index greater than 32 may be accessed, 0xffffffff is returned. -** -** It is not possible to determine if the old.rowid or new.rowid column is -** accessed by triggers. The caller must always assume that it is. -** -** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned -** applies to the old.* table. If 1, the new.* table. -** -** Parameter tr_tm must be a mask with one or both of the TRIGGER_BEFORE -** and TRIGGER_AFTER bits set. Values accessed by BEFORE triggers are only -** included in the returned mask if the TRIGGER_BEFORE bit is set in the -** tr_tm parameter. Similarly, values accessed by AFTER triggers are only -** included in the returned mask if the TRIGGER_AFTER bit is set in tr_tm. +** Insert code into "v" that will push the record on the top of the +** stack into the sorter. */ -SQLITE_PRIVATE u32 sqlite3TriggerColmask( - Parse *pParse, /* Parse context */ - Trigger *pTrigger, /* List of triggers on table pTab */ - ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ - int isNew, /* 1 for new.* ref mask, 0 for old.* ref mask */ - int tr_tm, /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ - Table *pTab, /* The table to code triggers from */ - int orconf /* Default ON CONFLICT policy for trigger steps */ +static void pushOntoSorter( + Parse *pParse, /* Parser context */ + ExprList *pOrderBy, /* The ORDER BY clause */ + Select *pSelect, /* The whole SELECT statement */ + int regData /* Register holding data to be sorted */ ){ - const int op = pChanges ? TK_UPDATE : TK_DELETE; - u32 mask = 0; - Trigger *p; - - assert( isNew==1 || isNew==0 ); - for(p=pTrigger; p; p=p->pNext){ - if( p->op==op && (tr_tm&p->tr_tm) - && checkColumnOverlap(p->pColumns,pChanges) - ){ - TriggerPrg *pPrg; - pPrg = getRowTrigger(pParse, p, pTab, orconf); - if( pPrg ){ - mask |= pPrg->aColmask[isNew]; - } + Vdbe *v = pParse->pVdbe; + int nExpr = pOrderBy->nExpr; + int regBase = sqlite3GetTempRange(pParse, nExpr+2); + int regRecord = sqlite3GetTempReg(pParse); + sqlite3ExprCacheClear(pParse); + sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + if( pSelect->iLimit ){ + int addr1, addr2; + int iLimit; + if( pSelect->iOffset ){ + iLimit = pSelect->iOffset+1; + }else{ + iLimit = pSelect->iLimit; } + addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); + sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); + addr2 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); + sqlite3VdbeJumpHere(v, addr2); } - - return mask; } -#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +/* +** Add code to implement the OFFSET +*/ +static void codeOffset( + Vdbe *v, /* Generate code into this VM */ + Select *p, /* The SELECT statement being coded */ + int iContinue /* Jump here to skip the current record */ +){ + if( p->iOffset && iContinue!=0 ){ + int addr; + sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); + VdbeComment((v, "skip OFFSET records")); + sqlite3VdbeJumpHere(v, addr); + } +} -/************** End of trigger.c *********************************************/ -/************** Begin file update.c ******************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Add code that will check to make sure the N registers starting at iMem +** form a distinct entry. iTab is a sorting index that holds previously +** seen combinations of the N values. A new entry is made in iTab +** if the current N values are new. ** -************************************************************************* -** This file contains C code routines that are called by the parser -** to handle UPDATE statements. +** A jump to addrRepeat is made and the N+1 values are popped from the +** stack if the top N elements are not distinct. */ +static void codeDistinct( + Parse *pParse, /* Parsing and code generating context */ + int iTab, /* A sorting index used to test for distinctness */ + int addrRepeat, /* Jump to here if not distinct */ + int N, /* Number of elements */ + int iMem /* First element */ +){ + Vdbe *v; + int r1; -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* Forward declaration */ -static void updateVirtualTable( - Parse *pParse, /* The parsing context */ - SrcList *pSrc, /* The virtual table to be modified */ - Table *pTab, /* The virtual table */ - ExprList *pChanges, /* The columns to change in the UPDATE statement */ - Expr *pRowidExpr, /* Expression used to recompute the rowid */ - int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ -); -#endif /* SQLITE_OMIT_VIRTUALTABLE */ + v = pParse->pVdbe; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); + sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); + sqlite3ReleaseTempReg(pParse, r1); +} +#ifndef SQLITE_OMIT_SUBQUERY /* -** The most recently coded instruction was an OP_Column to retrieve the -** i-th column of table pTab. This routine sets the P4 parameter of the -** OP_Column to the default value, if any. -** -** The default value of a column is specified by a DEFAULT clause in the -** column definition. This was either supplied by the user when the table -** was created, or added later to the table definition by an ALTER TABLE -** command. If the latter, then the row-records in the table btree on disk -** may not contain a value for the column and the default value, taken -** from the P4 parameter of the OP_Column instruction, is returned instead. -** If the former, then all row-records are guaranteed to include a value -** for the column and the P4 value is not required. -** -** Column definitions created by an ALTER TABLE command may only have -** literal default values specified: a number, null or a string. (If a more -** complicated default expression value was provided, it is evaluated -** when the ALTER TABLE is executed and one of the literal values written -** into the sqlite_master table.) -** -** Therefore, the P4 parameter is only required if the default value for -** the column is a literal number, string or null. The sqlite3ValueFromExpr() -** function is capable of transforming these types of expressions into -** sqlite3_value objects. -** -** If parameter iReg is not negative, code an OP_RealAffinity instruction -** on register iReg. This is used when an equivalent integer value is -** stored in place of an 8-byte floating point value in order to save -** space. +** Generate an error message when a SELECT is used within a subexpression +** (example: "a IN (SELECT * FROM table)") but it has more than 1 result +** column. We do this in a subroutine because the error used to occur +** in multiple places. (The error only occurs in one place now, but we +** retain the subroutine to minimize code disruption.) */ -SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ - assert( pTab!=0 ); - if( !pTab->pSelect ){ - sqlite3_value *pValue; - u8 enc = ENC(sqlite3VdbeDb(v)); - Column *pCol = &pTab->aCol[i]; - VdbeComment((v, "%s.%s", pTab->zName, pCol->zName)); - assert( inCol ); - sqlite3ValueFromExpr(sqlite3VdbeDb(v), pCol->pDflt, enc, - pCol->affinity, &pValue); - if( pValue ){ - sqlite3VdbeChangeP4(v, -1, (const char *)pValue, P4_MEM); - } -#ifndef SQLITE_OMIT_FLOATING_POINT - if( iReg>=0 && pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } -#endif +static int checkForMultiColumnSelectError( + Parse *pParse, /* Parse context. */ + SelectDest *pDest, /* Destination of SELECT results */ + int nExpr /* Number of result columns returned by SELECT */ +){ + int eDest = pDest->eDest; + if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ + sqlite3ErrorMsg(pParse, "only a single result allowed for " + "a SELECT that is part of an expression"); + return 1; + }else{ + return 0; } } +#endif /* -** Process an UPDATE statement. +** This routine generates the code for the inside of the inner loop +** of a SELECT. ** -** UPDATE OR IGNORE table_wxyz SET a=b, c=d WHERE e<5 AND f NOT NULL; -** \_______/ \________/ \______/ \________________/ -* onError pTabList pChanges pWhere +** If srcTab and nColumn are both zero, then the pEList expressions +** are evaluated in order to get the data for this row. If nColumn>0 +** then data is pulled from srcTab and pEList is used only to get the +** datatypes for each column. */ -SQLITE_PRIVATE void sqlite3Update( - Parse *pParse, /* The parser context */ - SrcList *pTabList, /* The table in which we should change things */ - ExprList *pChanges, /* Things to be changed */ - Expr *pWhere, /* The WHERE clause. May be null */ - int onError /* How to handle constraint errors */ +static void selectInnerLoop( + Parse *pParse, /* The parser context */ + Select *p, /* The complete select statement being coded */ + ExprList *pEList, /* List of values being extracted */ + int srcTab, /* Pull data from this table */ + int nColumn, /* Number of columns in the source table */ + ExprList *pOrderBy, /* If not NULL, sort results using this key */ + int distinct, /* If >=0, make sure results are distinct */ + SelectDest *pDest, /* How to dispose of the results */ + int iContinue, /* Jump here to continue with next row */ + int iBreak /* Jump here to break out of the inner loop */ ){ - int i, j; /* Loop counters */ - Table *pTab; /* The table to be updated */ - int addr = 0; /* VDBE instruction address of the start of the loop */ - WhereInfo *pWInfo; /* Information about the WHERE clause */ - Vdbe *v; /* The virtual database engine */ - Index *pIdx; /* For looping over indices */ - int nIdx; /* Number of indices that need updating */ - int iCur; /* VDBE Cursor number of pTab */ - sqlite3 *db; /* The database structure */ - int *aRegIdx = 0; /* One register assigned to each index to be updated */ - int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the - ** an expression for the i-th column of the table. - ** aXRef[i]==-1 if the i-th column is not changed. */ - int chngRowid; /* True if the record number is being changed */ - Expr *pRowidExpr = 0; /* Expression defining the new record number */ - int openAll = 0; /* True if all indices need to be opened */ - AuthContext sContext; /* The authorization context */ - NameContext sNC; /* The name-context to resolve expressions in */ - int iDb; /* Database containing the table being updated */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ - int hasFK; /* True if foreign key processing is required */ - -#ifndef SQLITE_OMIT_TRIGGER - int isView; /* True when updating a view (INSTEAD OF trigger) */ - Trigger *pTrigger; /* List of triggers on pTab, if required */ - int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ -#endif - int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */ - - /* Register Allocations */ - int regRowCount = 0; /* A count of rows changed */ - int regOldRowid; /* The old rowid */ - int regNewRowid; /* The new rowid */ - int regNew; - int regOld = 0; - int regRowSet = 0; /* Rowset of rows to be updated */ - int regRec; /* Register used for new table record to insert */ + Vdbe *v = pParse->pVdbe; + int i; + int hasDistinct; /* True if the DISTINCT keyword is present */ + int regResult; /* Start of memory holding result set */ + int eDest = pDest->eDest; /* How to dispose of results */ + int iParm = pDest->iParm; /* First argument to disposal method */ + int nResultCol; /* Number of result columns */ - memset(&sContext, 0, sizeof(sContext)); - db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ - goto update_cleanup; + assert( v ); + if( NEVER(v==0) ) return; + assert( pEList!=0 ); + hasDistinct = distinct>=0; + if( pOrderBy==0 && !hasDistinct ){ + codeOffset(v, p, iContinue); } - assert( pTabList->nSrc==1 ); - /* Locate the table which we want to update. - */ - pTab = sqlite3SrcListLookup(pParse, pTabList); - if( pTab==0 ) goto update_cleanup; - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - - /* Figure out if we have any triggers and if the table being - ** updated is a view. + /* Pull the requested columns. */ -#ifndef SQLITE_OMIT_TRIGGER - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask); - isView = pTab->pSelect!=0; - assert( pTrigger || tmask==0 ); -#else -# define pTrigger 0 -# define isView 0 -# define tmask 0 -#endif -#ifdef SQLITE_OMIT_VIEW -# undef isView -# define isView 0 -#endif - - if( sqlite3ViewGetColumnNames(pParse, pTab) ){ - goto update_cleanup; + if( nColumn>0 ){ + nResultCol = nColumn; + }else{ + nResultCol = pEList->nExpr; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ - goto update_cleanup; + if( pDest->iMem==0 ){ + pDest->iMem = pParse->nMem+1; + pDest->nMem = nResultCol; + pParse->nMem += nResultCol; + }else{ + assert( pDest->nMem==nResultCol ); } - aXRef = sqlite3DbMallocRaw(db, sizeof(int) * pTab->nCol ); - if( aXRef==0 ) goto update_cleanup; - for(i=0; inCol; i++) aXRef[i] = -1; + regResult = pDest->iMem; + if( nColumn>0 ){ + for(i=0; ia[0].iCursor = iCur = pParse->nTab++; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - pParse->nTab++; + if( hasDistinct ){ + assert( pEList!=0 ); + assert( pEList->nExpr==nColumn ); + codeDistinct(pParse, distinct, iContinue, nColumn, regResult); + if( pOrderBy==0 ){ + codeOffset(v, p, iContinue); + } } - /* Initialize the name-context */ - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - sNC.pSrcList = pTabList; + switch( eDest ){ + /* In this mode, write each query result to the key of the temporary + ** table iParm. + */ +#ifndef SQLITE_OMIT_COMPOUND_SELECT + case SRT_Union: { + int r1; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); + break; + } - /* Resolve the column names in all the expressions of the - ** of the UPDATE statement. Also find the column index - ** for each column to be updated in the pChanges array. For each - ** column to be updated, make sure we have authorization to change - ** that column. - */ - chngRowid = 0; - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ - goto update_cleanup; + /* Construct a record from the query result, but instead of + ** saving that record, use it as a key to delete elements from + ** the temporary table iParm. + */ + case SRT_Except: { + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); + break; } - for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ){ - if( j==pTab->iPKey ){ - chngRowid = 1; - pRowidExpr = pChanges->a[i].pExpr; - } - aXRef[j] = i; - break; +#endif + + /* Store the result as data using a unique key. + */ + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + if( pOrderBy ){ + pushOntoSorter(pParse, pOrderBy, p, r1); + }else{ + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); } + sqlite3ReleaseTempReg(pParse, r1); + break; } - if( j>=pTab->nCol ){ - if( sqlite3IsRowid(pChanges->a[i].zName) ){ - chngRowid = 1; - pRowidExpr = pChanges->a[i].pExpr; + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + assert( nColumn==1 ); + p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affinity); + if( pOrderBy ){ + /* At first glance you would think we could optimize out the + ** ORDER BY in this case since the order of entries in the set + ** does not matter. But there might be a LIMIT clause, in which + ** case the order does matter */ + pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ - sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); - pParse->checkSchema = 1; - goto update_cleanup; + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, regResult, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); } + break; } -#ifndef SQLITE_OMIT_AUTHORIZATION - { - int rc; - rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - pTab->aCol[j].zName, db->aDb[iDb].zName); - if( rc==SQLITE_DENY ){ - goto update_cleanup; - }else if( rc==SQLITE_IGNORE ){ - aXRef[j] = -1; + + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. + */ + case SRT_Mem: { + assert( nColumn==1 ); + if( pOrderBy ){ + pushOntoSorter(pParse, pOrderBy, p, regResult); + }else{ + sqlite3ExprCodeMove(pParse, regResult, iParm, 1); + /* The LIMIT clause will jump out of the loop for us */ + } + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + /* Send the data to the callback function or to a subroutine. In the + ** case of a subroutine, the subroutine itself is responsible for + ** popping the data from the stack. + */ + case SRT_Coroutine: + case SRT_Output: { + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + if( pOrderBy ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + pushOntoSorter(pParse, pOrderBy, p, r1); + sqlite3ReleaseTempReg(pParse, r1); + }else if( eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + }else{ + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); + sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); } + break; + } + +#if !defined(SQLITE_OMIT_TRIGGER) + /* Discard the results. This is used for SELECT statements inside + ** the body of a TRIGGER. The purpose of such selects is to call + ** user-defined functions that have side effects. We do not care + ** about the actual results of the select. + */ + default: { + assert( eDest==SRT_Discard ); + break; } #endif } - hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngRowid); - - /* Allocate memory for the array aRegIdx[]. There is one entry in the - ** array for each index associated with table being updated. Fill in - ** the value with a register number for indices that are to be used - ** and with zero for unused indices. + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. */ - for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} - if( nIdx>0 ){ - aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx ); - if( aRegIdx==0 ) goto update_cleanup; + if( pOrderBy==0 && p->iLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int reg; - if( chngRowid ){ - reg = ++pParse->nMem; - }else{ - reg = 0; - for(i=0; inColumn; i++){ - if( aXRef[pIdx->aiColumn[i]]>=0 ){ - reg = ++pParse->nMem; - break; - } +} + +/* +** Given an expression list, generate a KeyInfo structure that records +** the collating sequence for each expression in that expression list. +** +** If the ExprList is an ORDER BY or GROUP BY clause then the resulting +** KeyInfo structure is appropriate for initializing a virtual index to +** implement that clause. If the ExprList is the result set of a SELECT +** then the KeyInfo structure is appropriate for initializing a virtual +** index to implement a DISTINCT test. +** +** Space to hold the KeyInfo structure is obtain from malloc. The calling +** function is responsible for seeing that this structure is eventually +** freed. Add the KeyInfo structure to the P4 field of an opcode using +** P4_KEYINFO_HANDOFF is the usual way of dealing with this. +*/ +static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ + sqlite3 *db = pParse->db; + int nExpr; + KeyInfo *pInfo; + struct ExprList_item *pItem; + int i; + + nExpr = pList->nExpr; + pInfo = sqlite3DbMallocZero(db, sizeof(*pInfo) + nExpr*(sizeof(CollSeq*)+1) ); + if( pInfo ){ + pInfo->aSortOrder = (u8*)&pInfo->aColl[nExpr]; + pInfo->nField = (u16)nExpr; + pInfo->enc = ENC(db); + pInfo->db = db; + for(i=0, pItem=pList->a; ipExpr); + if( !pColl ){ + pColl = db->pDfltColl; } + pInfo->aColl[i] = pColl; + pInfo->aSortOrder[i] = pItem->sortOrder; } - aRegIdx[j] = reg; } + return pInfo; +} - /* Begin generating code. */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto update_cleanup; - if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); - sqlite3BeginWriteOperation(pParse, 1, iDb); - -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere); - pWhere = 0; - pTabList = 0; - goto update_cleanup; +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Name of the connection operator, used for error messages. +*/ +static const char *selectOpName(int id){ + char *z; + switch( id ){ + case TK_ALL: z = "UNION ALL"; break; + case TK_INTERSECT: z = "INTERSECT"; break; + case TK_EXCEPT: z = "EXCEPT"; break; + default: z = "UNION"; break; } -#endif + return z; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - /* Allocate required registers. */ - regOldRowid = regNewRowid = ++pParse->nMem; - if( pTrigger || hasFK ){ - regOld = pParse->nMem + 1; - pParse->nMem += pTab->nCol; - } - if( chngRowid || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of the form: +** +** "USE TEMP B-TREE FOR xxx" +** +** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which +** is determined by the zUsage argument. +*/ +static void explainTempTable(Parse *pParse, const char *zUsage){ + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); } - regNew = pParse->nMem + 1; - pParse->nMem += pTab->nCol; - regRec = ++pParse->nMem; +} - /* Start the view context. */ - if( isView ){ - sqlite3AuthContextPush(pParse, &sContext, pTab->zName); - } +/* +** Assign expression b to lvalue a. A second, no-op, version of this macro +** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code +** in sqlite3Select() to assign values to structure member variables that +** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the +** code with #ifndef directives. +*/ +# define explainSetInteger(a, b) a = b - /* If we are trying to update a view, realize that view into - ** a ephemeral table. - */ -#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) - if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iCur); - } +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainTempTable(y,z) +# define explainSetInteger(y,z) #endif - /* Resolve the column names in all the expressions in the - ** WHERE clause. - */ - if( sqlite3ResolveExprNames(&sNC, pWhere) ){ - goto update_cleanup; +#if !defined(SQLITE_OMIT_EXPLAIN) && !defined(SQLITE_OMIT_COMPOUND_SELECT) +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of one of the two forms: +** +** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)" +** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)" +** +** where iSub1 and iSub2 are the integers passed as the corresponding +** function parameters, and op is the text representation of the parameter +** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT, +** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is +** false, or the second form if it is true. +*/ +static void explainComposite( + Parse *pParse, /* Parse context */ + int op, /* One of TK_UNION, TK_EXCEPT etc. */ + int iSub1, /* Subquery id 1 */ + int iSub2, /* Subquery id 2 */ + int bUseTmp /* True if a temp table was used */ +){ + assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL ); + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf( + pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2, + bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op) + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); } +} +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainComposite(v,w,x,y,z) +#endif - /* Begin the database scan - */ - sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = pWInfo->okOnePass; +/* +** If the inner loop was generated using a non-null pOrderBy argument, +** then the results were placed in a sorter. After the loop is terminated +** we need to run the sorter and output the results. The following +** routine generates the code needed to do that. +*/ +static void generateSortTail( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + Vdbe *v, /* Generate code into this VDBE */ + int nColumn, /* Number of columns of data */ + SelectDest *pDest /* Write the sorted results here */ +){ + int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ + int addr; + int iTab; + int pseudoTab = 0; + ExprList *pOrderBy = p->pOrderBy; - /* Remember the rowid of every item to be updated. - */ - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid); - if( !okOnePass ){ - regRowSet = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); - } + int eDest = pDest->eDest; + int iParm = pDest->iParm; - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); + int regRow; + int regRowid; - /* Initialize the count of updated rows - */ - if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ - regRowCount = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + iTab = pOrderBy->iECursor; + regRow = sqlite3GetTempReg(pParse); + if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + pseudoTab = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); + regRowid = 0; + }else{ + regRowid = sqlite3GetTempReg(pParse); } - - if( !isView ){ - /* - ** Open every index that needs updating. Note that if any - ** index could potentially invoke a REPLACE conflict resolution - ** action, then we need to open all indices because we might need - ** to be deleting some records. - */ - if( !okOnePass ) sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenWrite); - if( onError==OE_Replace ){ - openAll = 1; - }else{ - openAll = 0; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_Replace ){ - openAll = 1; - break; + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow); + switch( eDest ){ + case SRT_Table: + case SRT_EphemTab: { + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case SRT_Set: { + assert( nColumn==1 ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, regRow, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); + break; + } + case SRT_Mem: { + assert( nColumn==1 ); + sqlite3ExprCodeMove(pParse, regRow, iParm, 1); + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + default: { + int i; + assert( eDest==SRT_Output || eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + testcase( eDest==SRT_Coroutine ); + for(i=0; iiMem+i ); + sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iMem+i); + if( i==0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); } } - } - for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - if( openAll || aRegIdx[i]>0 ){ - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iCur+i+1, pIdx->tnum, iDb, - (char*)pKey, P4_KEYINFO_HANDOFF); - assert( pParse->nTab>iCur+i+1 ); + if( eDest==SRT_Output ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iMem, nColumn); + sqlite3ExprCacheAffinityChange(pParse, pDest->iMem, nColumn); + }else{ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); } + break; } } + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); - /* Top of the update loop */ - if( okOnePass ){ - int a1 = sqlite3VdbeAddOp1(v, OP_NotNull, regOldRowid); - addr = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, a1); - }else{ - addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, 0, regOldRowid); + /* The bottom of the loop + */ + sqlite3VdbeResolveLabel(v, addrContinue); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + sqlite3VdbeResolveLabel(v, addrBreak); + if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); } +} - /* Make cursor iCur point to the record that is being updated. If - ** this record does not exist for some reason (deleted by a trigger, - ** for example, then jump to the next iteration of the RowSet loop. */ - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); - - /* If the record number will change, set register regNewRowid to - ** contain the new value. If the record number is not being modified, - ** then regNewRowid is the same register as regOldRowid, which is - ** already populated. */ - assert( chngRowid || pTrigger || hasFK || regOldRowid==regNewRowid ); - if( chngRowid ){ - sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); - sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); - } +/* +** Return a pointer to a string containing the 'declaration type' of the +** expression pExpr. The string may be treated as static by the caller. +** +** The declaration type is the exact datatype definition extracted from the +** original CREATE TABLE statement if the expression is a column. The +** declaration type for a ROWID field is INTEGER. Exactly when an expression +** is considered a column can be complex in the presence of subqueries. The +** result-set expression in all of the following SELECT statements is +** considered a column by this function. +** +** SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl); +** SELECT abc FROM (SELECT col AS abc FROM tbl); +** +** The declaration type for any expression other than a column is NULL. +*/ +static const char *columnType( + NameContext *pNC, + Expr *pExpr, + const char **pzOriginDb, + const char **pzOriginTab, + const char **pzOriginCol +){ + char const *zType = 0; + char const *zOriginDb = 0; + char const *zOriginTab = 0; + char const *zOriginCol = 0; + int j; + if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; - /* If there are triggers on this table, populate an array of registers - ** with the required old.* column data. */ - if( hasFK || pTrigger ){ - u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); - oldmask |= sqlite3TriggerColmask(pParse, - pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError - ); - for(i=0; inCol; i++){ - if( aXRef[i]<0 || oldmask==0xffffffff || (oldmask & (1<op ){ + case TK_AGG_COLUMN: + case TK_COLUMN: { + /* The expression is a column. Locate the table the column is being + ** extracted from in NameContext.pSrcList. This table may be real + ** database table or a subquery. + */ + Table *pTab = 0; /* Table structure column is extracted from */ + Select *pS = 0; /* Select the column is extracted from */ + int iCol = pExpr->iColumn; /* Index of column in pTab */ + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); + while( pNC && !pTab ){ + SrcList *pTabList = pNC->pSrcList; + for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); + if( jnSrc ){ + pTab = pTabList->a[j].pTab; + pS = pTabList->a[j].pSelect; + }else{ + pNC = pNC->pNext; + } } - } - if( chngRowid==0 ){ - sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); - } - } - /* Populate the array of registers beginning at regNew with the new - ** row data. This array is used to check constaints, create the new - ** table and index records, and as the values for any new.* references - ** made by triggers. - ** - ** If there are one or more BEFORE triggers, then do not populate the - ** registers associated with columns that are (a) not modified by - ** this UPDATE statement and (b) not accessed by new.* references. The - ** values for registers not modified by the UPDATE must be reloaded from - ** the database after the BEFORE triggers are fired anyway (as the trigger - ** may have modified them). So not loading those that are not going to - ** be used eliminates some redundant opcodes. - */ - newmask = sqlite3TriggerColmask( - pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError - ); - for(i=0; inCol; i++){ - if( i==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); - }else{ - j = aXRef[i]; - if( j>=0 ){ - sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); - }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask&(1<pTab==pTab ); + if( pS ){ + /* The "table" is actually a sub-select or a view in the FROM clause + ** of the SELECT statement. Return the declaration type and origin + ** data for the result-set column of the sub-select. */ - testcase( i==31 ); - testcase( i==32 ); - sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i); - sqlite3ColumnDefault(v, pTab, i, regNew+i); + if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ + /* If iCol is less than zero, then the expression requests the + ** rowid of the sub-select or view. This expression is legal (see + ** test case misc2.2.2) - it always evaluates to NULL. + */ + NameContext sNC; + Expr *p = pS->pEList->a[iCol].pExpr; + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); + } + }else if( ALWAYS(pTab->pSchema) ){ + /* A real table */ + assert( !pS ); + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==-1 || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zType = "INTEGER"; + zOriginCol = "rowid"; + }else{ + zType = pTab->aCol[iCol].zType; + zOriginCol = pTab->aCol[iCol].zName; + } + zOriginTab = pTab->zName; + if( pNC->pParse ){ + int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); + zOriginDb = pNC->pParse->db->aDb[iDb].zName; + } } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: { + /* The expression is a sub-select. Return the declaration type and + ** origin info for the single column in the result set of the SELECT + ** statement. + */ + NameContext sNC; + Select *pS = pExpr->x.pSelect; + Expr *p = pS->pEList->a[0].pExpr; + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); + break; } +#endif } + + if( pzOriginDb ){ + assert( pzOriginTab && pzOriginCol ); + *pzOriginDb = zOriginDb; + *pzOriginTab = zOriginTab; + *pzOriginCol = zOriginCol; + } + return zType; +} - /* Fire any BEFORE UPDATE triggers. This happens before constraints are - ** verified. One could argue that this is wrong. - */ - if( tmask&TRIGGER_BEFORE ){ - sqlite3VdbeAddOp2(v, OP_Affinity, regNew, pTab->nCol); - sqlite3TableAffinityStr(v, pTab); - sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, - TRIGGER_BEFORE, pTab, regOldRowid, onError, addr); - - /* The row-trigger may have deleted the row being updated. In this - ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behaviour - what happens when the row being updated - ** is deleted or renamed by a BEFORE trigger - is left undefined in the - ** documentation. - */ - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); +/* +** Generate code that will tell the VDBE the declaration types of columns +** in the result set. +*/ +static void generateColumnTypes( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* List of tables */ + ExprList *pEList /* Expressions defining the result set */ +){ +#ifndef SQLITE_OMIT_DECLTYPE + Vdbe *v = pParse->pVdbe; + int i; + NameContext sNC; + sNC.pSrcList = pTabList; + sNC.pParse = pParse; + for(i=0; inExpr; i++){ + Expr *p = pEList->a[i].pExpr; + const char *zType; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + const char *zOrigDb = 0; + const char *zOrigTab = 0; + const char *zOrigCol = 0; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); - /* If it did not delete it, the row-trigger may still have modified - ** some of the columns of the row being updated. Load the values for - ** all columns not modified by the update statement into their - ** registers in case this has happened. + /* The vdbe must make its own copy of the column-type and other + ** column specific strings, in case the schema is reset before this + ** virtual machine is deleted. */ - for(i=0; inCol; i++){ - if( aXRef[i]<0 && i!=pTab->iPKey ){ - sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i); - sqlite3ColumnDefault(v, pTab, i, regNew+i); - } - } + sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); +#else + zType = columnType(&sNC, p, 0, 0, 0); +#endif + sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } +#endif /* SQLITE_OMIT_DECLTYPE */ +} - if( !isView ){ - int j1; /* Address of jump instruction */ +/* +** Generate code that will tell the VDBE the names of columns +** in the result set. This information is used to provide the +** azCol[] values in the callback. +*/ +static void generateColumnNames( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* List of tables */ + ExprList *pEList /* Expressions defining the result set */ +){ + Vdbe *v = pParse->pVdbe; + int i, j; + sqlite3 *db = pParse->db; + int fullNames, shortNames; - /* Do constraint checks. */ - sqlite3GenerateConstraintChecks(pParse, pTab, iCur, regNewRowid, - aRegIdx, (chngRowid?regOldRowid:0), 1, onError, addr, 0); +#ifndef SQLITE_OMIT_EXPLAIN + /* If this is an EXPLAIN, skip this step */ + if( pParse->explain ){ + return; + } +#endif - /* Do FK constraint checks. */ - if( hasFK ){ - sqlite3FkCheck(pParse, pTab, regOldRowid, 0); + if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; + pParse->colNamesSet = 1; + fullNames = (db->flags & SQLITE_FullColNames)!=0; + shortNames = (db->flags & SQLITE_ShortColNames)!=0; + sqlite3VdbeSetNumCols(v, pEList->nExpr); + for(i=0; inExpr; i++){ + Expr *p; + p = pEList->a[i].pExpr; + if( NEVER(p==0) ) continue; + if( pEList->a[i].zName ){ + char *zName = pEList->a[i].zName; + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); + }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ + Table *pTab; + char *zCol; + int iCol = p->iColumn; + for(j=0; ALWAYS(jnSrc); j++){ + if( pTabList->a[j].iCursor==p->iTable ) break; + } + assert( jnSrc ); + pTab = pTabList->a[j].pTab; + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==-1 || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zCol = "rowid"; + }else{ + zCol = pTab->aCol[iCol].zName; + } + if( !shortNames && !fullNames ){ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, + sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); + }else if( fullNames ){ + char *zName = 0; + zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); + }else{ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); + } + }else{ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, + sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); } + } + generateColumnTypes(pParse, pTabList, pEList); +} - /* Delete the index entries associated with the current record. */ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regOldRowid); - sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, aRegIdx); - - /* If changing the record number, delete the old record. */ - if( hasFK || chngRowid ){ - sqlite3VdbeAddOp2(v, OP_Delete, iCur, 0); - } - sqlite3VdbeJumpHere(v, j1); +/* +** Given a an expression list (which is really the list of expressions +** that form the result set of a SELECT statement) compute appropriate +** column names for a table that would hold the expression list. +** +** All column names will be unique. +** +** Only the column names are computed. Column.zType, Column.zColl, +** and other fields of Column are zeroed. +** +** Return SQLITE_OK on success. If a memory allocation error occurs, +** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. +*/ +static int selectColumnsFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* Expr list from which to derive column names */ + int *pnCol, /* Write the number of columns here */ + Column **paCol /* Write the new column list here */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + int i, j; /* Loop counters */ + int cnt; /* Index added to make the name unique */ + Column *aCol, *pCol; /* For looping over result columns */ + int nCol; /* Number of columns in the result set */ + Expr *p; /* Expression for a single result column */ + char *zName; /* Column name */ + int nName; /* Size of name in zName[] */ - if( hasFK ){ - sqlite3FkCheck(pParse, pTab, 0, regNewRowid); + *pnCol = nCol = pEList->nExpr; + aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + if( aCol==0 ) return SQLITE_NOMEM; + for(i=0, pCol=aCol; ia[i].pExpr; + assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue) + || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 ); + if( (zName = pEList->a[i].zName)!=0 ){ + /* If the column contains an "AS " phrase, use as the name */ + zName = sqlite3DbStrDup(db, zName); + }else{ + Expr *pColExpr = p; /* The expression that is the result column name */ + Table *pTab; /* Table associated with this expression */ + while( pColExpr->op==TK_DOT ) pColExpr = pColExpr->pRight; + if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ + /* For columns use the column name name */ + int iCol = pColExpr->iColumn; + pTab = pColExpr->pTab; + if( iCol<0 ) iCol = pTab->iPKey; + zName = sqlite3MPrintf(db, "%s", + iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); + }else if( pColExpr->op==TK_ID ){ + assert( !ExprHasProperty(pColExpr, EP_IntValue) ); + zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken); + }else{ + /* Use the original text of the column expression as its name */ + zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan); + } + } + if( db->mallocFailed ){ + sqlite3DbFree(db, zName); + break; } - - /* Insert the new index entries and the new record. */ - sqlite3CompleteInsertion(pParse, pTab, iCur, regNewRowid, aRegIdx, 1, 0, 0); - /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to - ** handle rows (possibly in other tables) that refer via a foreign key - ** to the row just updated. */ - if( hasFK ){ - sqlite3FkActions(pParse, pTab, pChanges, regOldRowid); + /* Make sure the column name is unique. If the name is not unique, + ** append a integer to the name so that it becomes unique. + */ + nName = sqlite3Strlen30(zName); + for(j=cnt=0; jzName = zName; } - - /* Increment the row counter - */ - if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){ - sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + if( db->mallocFailed ){ + for(j=0; jdb; + NameContext sNC; + Column *pCol; + CollSeq *pColl; + int i; + Expr *p; + struct ExprList_item *a; - /* Close all tables */ - for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - if( openAll || aRegIdx[i]>0 ){ - sqlite3VdbeAddOp2(v, OP_Close, iCur+i+1, 0); + assert( pSelect!=0 ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); + assert( nCol==pSelect->pEList->nExpr || db->mallocFailed ); + if( db->mallocFailed ) return; + memset(&sNC, 0, sizeof(sNC)); + sNC.pSrcList = pSelect->pSrc; + a = pSelect->pEList->a; + for(i=0, pCol=aCol; izType = sqlite3DbStrDup(db, columnType(&sNC, p, 0, 0, 0)); + pCol->affinity = sqlite3ExprAffinity(p); + if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; + pColl = sqlite3ExprCollSeq(pParse, p); + if( pColl ){ + pCol->zColl = sqlite3DbStrDup(db, pColl->zName); } } - sqlite3VdbeAddOp2(v, OP_Close, iCur, 0); +} - /* Update the sqlite_sequence table by storing the content of the - ** maximum rowid counter values recorded while inserting into - ** autoincrement tables. - */ - if( pParse->nested==0 && pParse->pTriggerTab==0 ){ - sqlite3AutoincrementEnd(pParse); - } +/* +** Given a SELECT statement, generate a Table structure that describes +** the result set of that SELECT. +*/ +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ + Table *pTab; + sqlite3 *db = pParse->db; + int savedFlags; - /* - ** Return the number of rows that were changed. If this routine is - ** generating code because of a call to sqlite3NestedParse(), do not - ** invoke the callback function. - */ - if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC); + savedFlags = db->flags; + db->flags &= ~SQLITE_FullColNames; + db->flags |= SQLITE_ShortColNames; + sqlite3SelectPrep(pParse, pSelect, 0); + if( pParse->nErr ) return 0; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + db->flags = savedFlags; + pTab = sqlite3DbMallocZero(db, sizeof(Table) ); + if( pTab==0 ){ + return 0; } - -update_cleanup: - sqlite3AuthContextPop(&sContext); - sqlite3DbFree(db, aRegIdx); - sqlite3DbFree(db, aXRef); - sqlite3SrcListDelete(db, pTabList); - sqlite3ExprListDelete(db, pChanges); - sqlite3ExprDelete(db, pWhere); - return; + /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside + ** is disabled */ + assert( db->lookaside.bEnabled==0 ); + pTab->nRef = 1; + pTab->zName = 0; + pTab->nRowEst = 1000000; + selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); + pTab->iPKey = -1; + if( db->mallocFailed ){ + sqlite3DeleteTable(db, pTab); + return 0; + } + return pTab; } -/* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file -** (or in another file, if this file becomes part of the amalgamation). */ -#ifdef isView - #undef isView -#endif -#ifdef pTrigger - #undef pTrigger -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Generate code for an UPDATE of a virtual table. -** -** The strategy is that we create an ephemerial table that contains -** for each row to be changed: -** -** (A) The original rowid of that row. -** (B) The revised rowid for the row. (note1) -** (C) The content of every column in the row. -** -** Then we loop over this ephemeral table and for each row in -** the ephermeral table call VUpdate. -** -** When finished, drop the ephemeral table. -** -** (note1) Actually, if we know in advance that (A) is always the same -** as (B) we only store (A), then duplicate (A) when pulling -** it out of the ephemeral table before calling VUpdate. +** Get a VDBE for the given parser context. Create a new one if necessary. +** If an error occurs, return NULL and leave a message in pParse. */ -static void updateVirtualTable( - Parse *pParse, /* The parsing context */ - SrcList *pSrc, /* The virtual table to be modified */ - Table *pTab, /* The virtual table */ - ExprList *pChanges, /* The columns to change in the UPDATE statement */ - Expr *pRowid, /* Expression used to recompute the rowid */ - int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ -){ - Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ - ExprList *pEList = 0; /* The result set of the SELECT statement */ - Select *pSelect = 0; /* The SELECT statement */ - Expr *pExpr; /* Temporary expression */ - int ephemTab; /* Table holding the result of the SELECT */ - int i; /* Loop counter */ - int addr; /* Address of top of loop */ - int iReg; /* First register in set passed to OP_VUpdate */ - sqlite3 *db = pParse->db; /* Database connection */ - const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - SelectDest dest; - - /* Construct the SELECT statement that will find the new values for - ** all updated rows. - */ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); - if( pRowid ){ - pEList = sqlite3ExprListAppend(pParse, pEList, - sqlite3ExprDup(db, pRowid, 0)); - } - assert( pTab->iPKey<0 ); - for(i=0; inCol; i++){ - if( aXRef[i]>=0 ){ - pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); - }else{ - pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe; + if( v==0 ){ + v = pParse->pVdbe = sqlite3VdbeCreate(pParse->db); +#ifndef SQLITE_OMIT_TRACE + if( v ){ + sqlite3VdbeAddOp0(v, OP_Trace); } - pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); - } - pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); - - /* Create the ephemeral table into which the update results will - ** be stored. - */ - assert( v ); - ephemTab = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - - /* fill the ephemeral table - */ - sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); - sqlite3Select(pParse, pSelect, &dest); - - /* Generate code to scan the ephemeral table and call VUpdate. */ - iReg = ++pParse->nMem; - pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); - for(i=0; inCol; i++){ - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); +#endif } - sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); - sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - - /* Cleanup */ - sqlite3SelectDelete(db, pSelect); + return v; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -/************** End of update.c **********************************************/ -/************** Begin file vacuum.c ******************************************/ + /* -** 2003 April 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code used to implement the VACUUM command. +** Compute the iLimit and iOffset fields of the SELECT based on the +** pLimit and pOffset expressions. pLimit and pOffset hold the expressions +** that appear in the original SQL statement after the LIMIT and OFFSET +** keywords. Or NULL if those keywords are omitted. iLimit and iOffset +** are the integer memory register numbers for counters used to compute +** the limit and offset. If there is no limit and/or offset, then +** iLimit and iOffset are negative. ** -** Most of the code in this file may be omitted by defining the -** SQLITE_OMIT_VACUUM macro. +** This routine changes the values of iLimit and iOffset only if +** a limit or offset is defined by pLimit and pOffset. iLimit and +** iOffset should have been preset to appropriate default values +** (usually but not always -1) prior to calling this routine. +** Only if pLimit!=0 or pOffset!=0 do the limit registers get +** redefined. The UNION ALL operator uses this property to force +** the reuse of the same limit and offset registers across multiple +** SELECT statements. */ +static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ + Vdbe *v = 0; + int iLimit = 0; + int iOffset; + int addr1, n; + if( p->iLimit ) return; -#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* -** Finalize a prepared statement. If there was an error, store the -** text of the error message in *pzErrMsg. Return the result code. -*/ -static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ - int rc; - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - if( rc ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + /* + ** "LIMIT -1" always shows all rows. There is some + ** contraversy about what the correct behavior should be. + ** The current implementation interprets "LIMIT 0" to mean + ** no rows. + */ + sqlite3ExprCacheClear(pParse); + assert( p->pOffset==0 || p->pLimit!=0 ); + if( p->pLimit ){ + p->iLimit = iLimit = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ + if( sqlite3ExprIsInteger(p->pLimit, &n) ){ + sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); + VdbeComment((v, "LIMIT counter")); + if( n==0 ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + }else{ + if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n; + } + }else{ + sqlite3ExprCode(pParse, p->pLimit, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); + VdbeComment((v, "LIMIT counter")); + sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); + } + if( p->pOffset ){ + p->iOffset = iOffset = ++pParse->nMem; + pParse->nMem++; /* Allocate an extra register for limit+offset */ + sqlite3ExprCode(pParse, p->pOffset, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); + VdbeComment((v, "OFFSET counter")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); + VdbeComment((v, "LIMIT+OFFSET")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); + sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); + sqlite3VdbeJumpHere(v, addr1); + } } - return rc; } +#ifndef SQLITE_OMIT_COMPOUND_SELECT /* -** Execute zSql on database db. Return an error code. +** Return the appropriate collating sequence for the iCol-th column of +** the result set for the compound-select statement "p". Return NULL if +** the column has no default collating sequence. +** +** The collating sequence for the compound select is taken from the +** left-most term of the select that has a collating sequence. */ -static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - VVA_ONLY( int rc; ) - if( !zSql ){ - return SQLITE_NOMEM; +static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ + CollSeq *pRet; + if( p->pPrior ){ + pRet = multiSelectCollSeq(pParse, p->pPrior, iCol); + }else{ + pRet = 0; } - if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - return sqlite3_errcode(db); + assert( iCol>=0 ); + if( pRet==0 && iColpEList->nExpr ){ + pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); } - VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW ); - return vacuumFinalize(db, pStmt, pzErrMsg); + return pRet; } +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/* -** Execute zSql on database db. The statement returns exactly -** one column. Execute this as SQL on the same database. -*/ -static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - int rc; - - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); - if( rc!=SQLITE_OK ){ - vacuumFinalize(db, pStmt, pzErrMsg); - return rc; - } - } +/* Forward reference */ +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +); - return vacuumFinalize(db, pStmt, pzErrMsg); -} +#ifndef SQLITE_OMIT_COMPOUND_SELECT /* -** The non-standard VACUUM command is used to clean up the database, -** collapse free space, etc. It is modelled after the VACUUM command -** in PostgreSQL. +** This routine is called to process a compound query form from +** two or more separate queries using UNION, UNION ALL, EXCEPT, or +** INTERSECT ** -** In version 1.0.x of SQLite, the VACUUM command would call -** gdbm_reorganize() on all the database tables. But beginning -** with 2.0.0, SQLite no longer uses GDBM so this command has -** become a no-op. -*/ -SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ - Vdbe *v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); - } - return; -} - -/* -** This routine implements the OP_Vacuum opcode of the VDBE. +** "p" points to the right-most of the two queries. the query on the +** left is p->pPrior. The left query could also be a compound query +** in which case this routine will be called recursively. +** +** The results of the total query are to be written into a destination +** of type eDest with parameter iParm. +** +** Example 1: Consider a three-way compound SQL statement. +** +** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3 +** +** This statement is parsed up as follows: +** +** SELECT c FROM t3 +** | +** `-----> SELECT b FROM t2 +** | +** `------> SELECT a FROM t1 +** +** The arrows in the diagram above represent the Select.pPrior pointer. +** So if this routine is called with p equal to the t3 query, then +** pPrior will be the t2 query. p->op will be TK_UNION in this case. +** +** Notice that because of the way SQLite parses compound SELECTs, the +** individual selects always group from left to right. */ -SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ - int rc = SQLITE_OK; /* Return code from service routines */ - Btree *pMain; /* The database being vacuumed */ - Btree *pTemp; /* The temporary database we vacuum into */ - char *zSql = 0; /* SQL statements */ - int saved_flags; /* Saved value of the db->flags */ - int saved_nChange; /* Saved value of db->nChange */ - int saved_nTotalChange; /* Saved value of db->nTotalChange */ - void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ - Db *pDb = 0; /* Database to detach at end of vacuum */ - int isMemDb; /* True if vacuuming a :memory: database */ - int nRes; /* Bytes of reserved space at the end of each page */ - int nDb; /* Number of attached databases */ +static int multiSelect( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int rc = SQLITE_OK; /* Success code from a subroutine */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest dest; /* Alternative data destination */ + Select *pDelete = 0; /* Chain of simple selects to delete */ + sqlite3 *db; /* Database connection */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif - if( !db->autoCommit ){ - sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); - return SQLITE_ERROR; + /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only + ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. + */ + assert( p && p->pPrior ); /* Calling function guarantees this much */ + db = pParse->db; + pPrior = p->pPrior; + assert( pPrior->pRightmost!=pPrior ); + assert( pPrior->pRightmost==p->pRightmost ); + dest = *pDest; + if( pPrior->pOrderBy ){ + sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", + selectOpName(p->op)); + rc = 1; + goto multi_select_end; } - if( db->activeVdbeCnt>1 ){ - sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); - return SQLITE_ERROR; + if( pPrior->pLimit ){ + sqlite3ErrorMsg(pParse,"LIMIT clause should come after %s not before", + selectOpName(p->op)); + rc = 1; + goto multi_select_end; } - /* Save the current value of the database flags so that it can be - ** restored before returning. Then set the writable-schema flag, and - ** disable CHECK and foreign key constraints. */ - saved_flags = db->flags; - saved_nChange = db->nChange; - saved_nTotalChange = db->nTotalChange; - saved_xTrace = db->xTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); - db->xTrace = 0; - - pMain = db->aDb[0].pBt; - isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); /* The VDBE already created by calling function */ - /* Attach the temporary database as 'vacuum_db'. The synchronous pragma - ** can be set to 'off' for this file, as it is not recovered if a crash - ** occurs anyway. The integrity of the database is maintained by a - ** (possibly synchronous) transaction opened on the main database before - ** sqlite3BtreeCopyFile() is called. - ** - ** An optimisation would be to use a non-journaled pager. - ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but - ** that actually made the VACUUM run slower. Very little journalling - ** actually occurs when doing a vacuum since the vacuum_db is initially - ** empty. Only the journal header is written. Apparently it takes more - ** time to parse and run the PRAGMA to turn journalling off than it does - ** to write the journal header file. + /* Create the destination temporary table if necessary */ - nDb = db->nDb; - if( sqlite3TempInMemory(db) ){ - zSql = "ATTACH ':memory:' AS vacuum_db;"; - }else{ - zSql = "ATTACH '' AS vacuum_db;"; - } - rc = execSql(db, pzErrMsg, zSql); - if( db->nDb>nDb ){ - pDb = &db->aDb[db->nDb-1]; - assert( strcmp(pDb->zName,"vacuum_db")==0 ); + if( dest.eDest==SRT_EphemTab ){ + assert( p->pEList ); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + dest.eDest = SRT_Table; } - if( rc!=SQLITE_OK ) goto end_of_vacuum; - pTemp = db->aDb[db->nDb-1].pBt; - - /* The call to execSql() to attach the temp database has left the file - ** locked (as there was more than one active statement when the transaction - ** to read the schema was concluded. Unlock it here so that this doesn't - ** cause problems for the call to BtreeSetPageSize() below. */ - sqlite3BtreeCommit(pTemp); - - nRes = sqlite3BtreeGetReserve(pMain); - /* A VACUUM cannot change the pagesize of an encrypted database. */ -#ifdef SQLITE_HAS_CODEC - if( db->nextPagesize ){ - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); - int nKey; - char *zKey; - sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey ) db->nextPagesize = 0; + /* Make sure all SELECTs in the statement have the same number of elements + ** in their result sets. + */ + assert( p->pEList && pPrior->pEList ); + if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + rc = 1; + goto multi_select_end; } -#endif - /* Do not attempt to change the page size for a WAL database */ - if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) - ==PAGER_JOURNALMODE_WAL ){ - db->nextPagesize = 0; + /* Compound SELECTs that have an ORDER BY clause are handled separately. + */ + if( p->pOrderBy ){ + return multiSelectOrderBy(pParse, p, pDest); } - if( sqlite3BtreeSetPageSize(pTemp, sqlite3BtreeGetPageSize(pMain), nRes, 0) - || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) - || NEVER(db->mallocFailed) - ){ - rc = SQLITE_NOMEM; - goto end_of_vacuum; - } - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ){ - goto end_of_vacuum; - } + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + int nLimit; + assert( !pPrior->pLimit ); + pPrior->pLimit = p->pLimit; + pPrior->pOffset = p->pOffset; + explainSetInteger(iSub1, pParse->iNextSelectId); + rc = sqlite3Select(pParse, pPrior, &dest); + p->pLimit = 0; + p->pOffset = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); + VdbeComment((v, "Jump ahead if LIMIT reached")); + } + explainSetInteger(iSub2, pParse->iNextSelectId); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow += pPrior->nSelectRow; + if( pPrior->pLimit + && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) + && p->nSelectRow > (double)nLimit + ){ + p->nSelectRow = (double)nLimit; + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temporary table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ + int addr; + SelectDest uniondest; -#ifndef SQLITE_OMIT_AUTOVACUUM - sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : - sqlite3BtreeGetAutoVacuum(pMain)); -#endif + testcase( p->op==TK_EXCEPT ); + testcase( p->op==TK_UNION ); + priorOp = SRT_Union; + if( dest.eDest==priorOp && ALWAYS(!p->pLimit &&!p->pOffset) ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pRightmost!=p ); /* Can only happen for leftward elements + ** of a 3-way or more compound */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + assert( p->pOffset==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + p->pRightmost->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } - /* Begin a transaction */ - rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Code the SELECT statements to our left + */ + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + explainSetInteger(iSub1, pParse->iNextSelectId); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } - /* Query the schema of the main database. Create a mirror schema - ** in the temporary database. - */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " - " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" - " AND rootpage>0" - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" - " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " - " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - - /* Loop through the tables in the main database. For each, do - ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy - ** the contents to the temporary database. - */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'INSERT INTO vacuum_db.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';'" - "FROM main.sqlite_master " - "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND rootpage>0" - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - - /* Copy over the sequence table - */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " - "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'INSERT INTO vacuum_db.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';' " - "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Code the current SELECT statement + */ + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + pOffset = p->pOffset; + p->pOffset = 0; + uniondest.eDest = op; + explainSetInteger(iSub2, pParse->iNextSelectId); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); + /* Query flattening in sqlite3Select() might refill p->pOrderBy. + ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ + sqlite3ExprListDelete(db, p->pOrderBy); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->pOffset = pOffset; + p->iLimit = 0; + p->iOffset = 0; + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + assert( unionTab==dest.iParm || dest.eDest!=priorOp ); + if( dest.eDest!=priorOp ){ + int iCont, iBreak, iStart; + assert( p->pEList ); + if( dest.eDest==SRT_Output ){ + Select *pFirst = p; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, + 0, -1, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + default: assert( p->op==TK_INTERSECT ); { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit, *pOffset; + int addr; + SelectDest intersectdest; + int r1; - /* Copy the triggers, views, and virtual tables from the main database - ** over to the temporary database. None of these objects has any - ** associated storage, so all we have to do is copy their entries - ** from the SQLITE_MASTER table. - */ - rc = execSql(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_master " - " SELECT type, name, tbl_name, rootpage, sql" - " FROM main.sqlite_master" - " WHERE type='view' OR type='trigger'" - " OR (type='table' AND rootpage=0)" - ); - if( rc ) goto end_of_vacuum; + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); - /* At this point, unless the main db was completely empty, there is now a - ** transaction open on the vacuum database, but not on the main database. - ** Open a btree level transaction on the main database. This allows a - ** call to sqlite3BtreeCopyFile(). The main database btree level - ** transaction is then committed, so the SQL level never knows it was - ** opened for writing. This way, the SQL transaction used to create the - ** temporary database never needs to be committed. - */ - { - u32 meta; - int i; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + p->pRightmost->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); - /* This array determines which meta meta values are preserved in the - ** vacuum. Even entries are the meta value number and odd entries - ** are an increment to apply to the meta value after the vacuum. - ** The increment is used to increase the schema cookie so that other - ** connections to the same database will know to reread the schema. - */ - static const unsigned char aCopy[] = { - BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ - BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ - BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ - BTREE_USER_VERSION, 0, /* Preserve the user version */ - }; + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + explainSetInteger(iSub1, pParse->iNextSelectId); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; + } - assert( 1==sqlite3BtreeIsInTrans(pTemp) ); - assert( 1==sqlite3BtreeIsInTrans(pMain) ); + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + pOffset = p->pOffset; + p->pOffset = 0; + intersectdest.iParm = tab2; + explainSetInteger(iSub2, pParse->iNextSelectId); + rc = sqlite3Select(pParse, p, &intersectdest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->pOffset = pOffset; - /* Copy Btree meta values */ - for(i=0; ipEList ); + if( dest.eDest==SRT_Output ){ + Select *pFirst = p; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, + 0, -1, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; } - - rc = sqlite3BtreeCopyFile(pMain, pTemp); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeCommit(pTemp); - if( rc!=SQLITE_OK ) goto end_of_vacuum; -#ifndef SQLITE_OMIT_AUTOVACUUM - sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp)); -#endif } - assert( rc==SQLITE_OK ); - rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1); - -end_of_vacuum: - /* Restore the original value of db->flags */ - db->flags = saved_flags; - db->nChange = saved_nChange; - db->nTotalChange = saved_nTotalChange; - db->xTrace = saved_xTrace; - sqlite3BtreeSetPageSize(pMain, -1, -1, 1); + explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL); - /* Currently there is an SQL level transaction open on the vacuum - ** database. No locks are held on any other files (since the main file - ** was committed at the btree level). So it safe to end the transaction - ** by manually setting the autoCommit flag to true and detaching the - ** vacuum database. The vacuum_db journal file is deleted when the pager - ** is closed by the DETACH. + /* Compute collating sequences used by + ** temporary tables needed to implement the compound select. + ** Attach the KeyInfo structure to all temporary tables. + ** + ** This section is run by the right-most SELECT statement only. + ** SELECT statements to the left always skip this part. The right-most + ** SELECT might also skip this part if it has no ORDER BY clause and + ** no temp tables are required. */ - db->autoCommit = 1; + if( p->selFlags & SF_UsesEphemeral ){ + int i; /* Loop counter */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + Select *pLoop; /* For looping through SELECT statements */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + int nCol; /* Number of columns in result set */ - if( pDb ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - pDb->pSchema = 0; - } + assert( p->pRightmost==p ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3DbMallocZero(db, + sizeof(*pKeyInfo)+nCol*(sizeof(CollSeq*) + 1)); + if( !pKeyInfo ){ + rc = SQLITE_NOMEM; + goto multi_select_end; + } + + pKeyInfo->enc = ENC(db); + pKeyInfo->nField = (u16)nCol; - sqlite3ResetInternalSchema(db, 0); + for(i=0, apColl=pKeyInfo->aColl; ipDfltColl; + } + } + + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + for(i=0; i<2; i++){ + int addr = pLoop->addrOpenEphm[i]; + if( addr<0 ){ + /* If [0] is unused then [1] is also unused. So we can + ** always safely abort as soon as the first unused slot is found */ + assert( pLoop->addrOpenEphm[1]<0 ); + break; + } + sqlite3VdbeChangeP2(v, addr, nCol); + sqlite3VdbeChangeP4(v, addr, (char*)pKeyInfo, P4_KEYINFO); + pLoop->addrOpenEphm[i] = -1; + } + } + sqlite3DbFree(db, pKeyInfo); + } +multi_select_end: + pDest->iMem = dest.iMem; + pDest->nMem = dest.nMem; + sqlite3SelectDelete(db, pDelete); return rc; } -#endif /* SQLITE_OMIT_VACUUM && SQLITE_OMIT_ATTACH */ +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/************** End of vacuum.c **********************************************/ -/************** Begin file vtab.c ********************************************/ /* -** 2006 June 10 +** Code an output subroutine for a coroutine implementation of a +** SELECT statment. ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** The data to be output is contained in pIn->iMem. There are +** pIn->nMem columns to be output. pDest is where the output should +** be sent. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** regReturn is the number of the register holding the subroutine +** return address. ** -************************************************************************* -** This file contains code used to help implement virtual tables. -*/ -#ifndef SQLITE_OMIT_VIRTUALTABLE - -/* -** The actual function that does the work of creating a new module. -** This function implements the sqlite3_create_module() and -** sqlite3_create_module_v2() interfaces. +** If regPrev>0 then it is the first register in a vector that +** records the previous output. mem[regPrev] is a flag that is false +** if there has been no previous output. If regPrev>0 then code is +** generated to suppress duplicates. pKeyInfo is used for comparing +** keys. +** +** If the LIMIT found in p->iLimit is reached, jump immediately to +** iBreak. */ -static int createModule( - sqlite3 *db, /* Database in which module is registered */ - const char *zName, /* Name assigned to this module */ - const sqlite3_module *pModule, /* The definition of the module */ - void *pAux, /* Context pointer for xCreate/xConnect */ - void (*xDestroy)(void *) /* Module destructor function */ +static int generateOutputSubroutine( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SelectDest *pIn, /* Coroutine supplying data */ + SelectDest *pDest, /* Where to send the data */ + int regReturn, /* The return address register */ + int regPrev, /* Previous result register. No uniqueness if 0 */ + KeyInfo *pKeyInfo, /* For comparing with previous entry */ + int p4type, /* The p4 type for pKeyInfo */ + int iBreak /* Jump here if we hit the LIMIT */ ){ - int rc, nName; - Module *pMod; + Vdbe *v = pParse->pVdbe; + int iContinue; + int addr; - sqlite3_mutex_enter(db->mutex); - nName = sqlite3Strlen30(zName); - pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); - if( pMod ){ - Module *pDel; - char *zCopy = (char *)(&pMod[1]); - memcpy(zCopy, zName, nName+1); - pMod->zName = zCopy; - pMod->pModule = pModule; - pMod->pAux = pAux; - pMod->xDestroy = xDestroy; - pDel = (Module *)sqlite3HashInsert(&db->aModule, zCopy, nName, (void*)pMod); - if( pDel && pDel->xDestroy ){ - pDel->xDestroy(pDel->pAux); - } - sqlite3DbFree(db, pDel); - if( pDel==pMod ){ - db->mallocFailed = 1; - } - sqlite3ResetInternalSchema(db, 0); - }else if( xDestroy ){ - xDestroy(pAux); + addr = sqlite3VdbeCurrentAddr(v); + iContinue = sqlite3VdbeMakeLabel(v); + + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + */ + if( regPrev ){ + int j1, j2; + j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); + j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iMem, regPrev+1, pIn->nMem, + (char*)pKeyInfo, p4type); + sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); + sqlite3VdbeJumpHere(v, j1); + sqlite3ExprCodeCopy(pParse, pIn->iMem, regPrev+1, pIn->nMem); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } - rc = sqlite3ApiExit(db, SQLITE_OK); - sqlite3_mutex_leave(db->mutex); - return rc; -} + if( pParse->db->mallocFailed ) return 0; + /* Suppress the the first OFFSET entries if there is an OFFSET clause + */ + codeOffset(v, p, iContinue); -/* -** External API function used to create a new virtual-table module. -*/ -SQLITE_API int sqlite3_create_module( - sqlite3 *db, /* Database in which module is registered */ - const char *zName, /* Name assigned to this module */ - const sqlite3_module *pModule, /* The definition of the module */ - void *pAux /* Context pointer for xCreate/xConnect */ -){ - return createModule(db, zName, pModule, pAux, 0); -} + switch( pDest->eDest ){ + /* Store the result as data using a unique key. + */ + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + testcase( pDest->eDest==SRT_Table ); + testcase( pDest->eDest==SRT_EphemTab ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iMem, pIn->nMem, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + sqlite3ReleaseTempReg(pParse, r1); + break; + } -/* -** External API function used to create a new virtual-table module. -*/ -SQLITE_API int sqlite3_create_module_v2( - sqlite3 *db, /* Database in which module is registered */ - const char *zName, /* Name assigned to this module */ - const sqlite3_module *pModule, /* The definition of the module */ - void *pAux, /* Context pointer for xCreate/xConnect */ - void (*xDestroy)(void *) /* Module destructor function */ -){ - return createModule(db, zName, pModule, pAux, xDestroy); -} +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + int r1; + assert( pIn->nMem==1 ); + p->affinity = + sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affinity); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iMem, 1, r1, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); + break; + } -/* -** Lock the virtual table so that it cannot be disconnected. -** Locks nest. Every lock should have a corresponding unlock. -** If an unlock is omitted, resources leaks will occur. -** -** If a disconnect is attempted while a virtual table is locked, -** the disconnect is deferred until all locks have been removed. -*/ -SQLITE_PRIVATE void sqlite3VtabLock(VTable *pVTab){ - pVTab->nRef++; -} +#if 0 /* Never occurs on an ORDER BY query */ + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. + */ + case SRT_Mem: { + assert( pIn->nMem==1 ); + sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iParm, 1); + /* The LIMIT clause will jump out of the loop for us */ + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ -/* -** pTab is a pointer to a Table structure representing a virtual-table. -** Return a pointer to the VTable object used by connection db to access -** this virtual-table, if one has been created, or NULL otherwise. -*/ -SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3 *db, Table *pTab){ - VTable *pVtab; - assert( IsVirtual(pTab) ); - for(pVtab=pTab->pVTable; pVtab && pVtab->db!=db; pVtab=pVtab->pNext); - return pVtab; -} + /* The results are stored in a sequence of registers + ** starting at pDest->iMem. Then the co-routine yields. + */ + case SRT_Coroutine: { + if( pDest->iMem==0 ){ + pDest->iMem = sqlite3GetTempRange(pParse, pIn->nMem); + pDest->nMem = pIn->nMem; + } + sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iMem, pDest->nMem); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + break; + } -/* -** Decrement the ref-count on a virtual table object. When the ref-count -** reaches zero, call the xDisconnect() method to delete the object. -*/ -SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ - sqlite3 *db = pVTab->db; - - assert( db ); - assert( pVTab->nRef>0 ); - assert( sqlite3SafetyCheckOk(db) ); - - pVTab->nRef--; - if( pVTab->nRef==0 ){ - sqlite3_vtab *p = pVTab->pVtab; - if( p ){ - p->pModule->xDisconnect(p); + /* If none of the above, then the result destination must be + ** SRT_Output. This routine is never called with any other + ** destination other than the ones handled above or SRT_Output. + ** + ** For SRT_Output, results are stored in a sequence of registers. + ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to + ** return the next row of result. + */ + default: { + assert( pDest->eDest==SRT_Output ); + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iMem, pIn->nMem); + sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, pIn->nMem); + break; } - sqlite3DbFree(db, pVTab); } -} - -/* -** Table p is a virtual table. This function moves all elements in the -** p->pVTable list to the sqlite3.pDisconnect lists of their associated -** database connections to be disconnected at the next opportunity. -** Except, if argument db is not NULL, then the entry associated with -** connection db is left in the p->pVTable list. -*/ -static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ - VTable *pRet = 0; - VTable *pVTable = p->pVTable; - p->pVTable = 0; - - /* Assert that the mutex (if any) associated with the BtShared database - ** that contains table p is held by the caller. See header comments - ** above function sqlite3VtabUnlockList() for an explanation of why - ** this makes it safe to access the sqlite3.pDisconnect list of any - ** database connection that may have an entry in the p->pVTable list. */ - assert( db==0 || - sqlite3BtreeHoldsMutex(db->aDb[sqlite3SchemaToIndex(db, p->pSchema)].pBt) - ); - while( pVTable ){ - sqlite3 *db2 = pVTable->db; - VTable *pNext = pVTable->pNext; - assert( db2 ); - if( db2==db ){ - pRet = pVTable; - p->pVTable = pRet; - pRet->pNext = 0; - }else{ - pVTable->pNext = db2->pDisconnect; - db2->pDisconnect = pVTable; - } - pVTable = pNext; + /* Jump to the end of the loop if the LIMIT is reached. + */ + if( p->iLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } - assert( !db || pRet ); - return pRet; -} + /* Generate the subroutine return + */ + sqlite3VdbeResolveLabel(v, iContinue); + sqlite3VdbeAddOp1(v, OP_Return, regReturn); + return addr; +} /* -** Disconnect all the virtual table objects in the sqlite3.pDisconnect list. +** Alternative compound select code generator for cases when there +** is an ORDER BY clause. ** -** This function may only be called when the mutexes associated with all -** shared b-tree databases opened using connection db are held by the -** caller. This is done to protect the sqlite3.pDisconnect list. The -** sqlite3.pDisconnect list is accessed only as follows: +** We assume a query of the following form: ** -** 1) By this function. In this case, all BtShared mutexes and the mutex -** associated with the database handle itself must be held. +** ORDER BY ** -** 2) By function vtabDisconnectAll(), when it adds a VTable entry to -** the sqlite3.pDisconnect list. In this case either the BtShared mutex -** associated with the database the virtual table is stored in is held -** or, if the virtual table is stored in a non-sharable database, then -** the database handle mutex is held. +** is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea +** is to code both and with the ORDER BY clause as +** co-routines. Then run the co-routines in parallel and merge the results +** into the output. In addition to the two coroutines (called selectA and +** selectB) there are 7 subroutines: ** -** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously -** by multiple threads. It is thread-safe. +** outA: Move the output of the selectA coroutine into the output +** of the compound query. +** +** outB: Move the output of the selectB coroutine into the output +** of the compound query. (Only generated for UNION and +** UNION ALL. EXCEPT and INSERTSECT never output a row that +** appears only in B.) +** +** AltB: Called when there is data from both coroutines and AB. +** +** EofA: Called when data is exhausted from selectA. +** +** EofB: Called when data is exhausted from selectB. +** +** The implementation of the latter five subroutines depend on which +** is used: +** +** +** UNION ALL UNION EXCEPT INTERSECT +** ------------- ----------------- -------------- ----------------- +** AltB: outA, nextA outA, nextA outA, nextA nextA +** +** AeqB: outA, nextA nextA nextA outA, nextA +** +** AgtB: outB, nextB outB, nextB nextB nextB +** +** EofA: outB, nextB outB, nextB halt halt +** +** EofB: outA, nextA outA, nextA outA, nextA halt +** +** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA +** causes an immediate jump to EofA and an EOF on B following nextB causes +** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or +** following nextX causes a jump to the end of the select processing. +** +** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled +** within the output subroutine. The regPrev register set holds the previously +** output value. A comparison is made against this value and the output +** is skipped if the next results would be the same as the previous. +** +** The implementation plan is to implement the two coroutines and seven +** subroutines first, then put the control logic at the bottom. Like this: +** +** goto Init +** coA: coroutine for left query (A) +** coB: coroutine for right query (B) +** outA: output one row of A +** outB: output one row of B (UNION and UNION ALL only) +** EofA: ... +** EofB: ... +** AltB: ... +** AeqB: ... +** AgtB: ... +** Init: initialize coroutine registers +** yield coA +** if eof(A) goto EofA +** yield coB +** if eof(B) goto EofB +** Cmpr: Compare A, B +** Jump AltB, AeqB, AgtB +** End: ... +** +** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not +** actually called using Gosub and they do not Return. EofA and EofB loop +** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** and AgtB jump to either L2 or to one of EofA or EofB. */ -SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ - VTable *p = db->pDisconnect; - db->pDisconnect = 0; +#ifndef SQLITE_OMIT_COMPOUND_SELECT +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int i, j; /* Loop counters */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest destA; /* Destination for coroutine A */ + SelectDest destB; /* Destination for coroutine B */ + int regAddrA; /* Address register for select-A coroutine */ + int regEofA; /* Flag to indicate when select-A is complete */ + int regAddrB; /* Address register for select-B coroutine */ + int regEofB; /* Flag to indicate when select-B is complete */ + int addrSelectA; /* Address of the select-A coroutine */ + int addrSelectB; /* Address of the select-B coroutine */ + int regOutA; /* Address register for the output-A subroutine */ + int regOutB; /* Address register for the output-B subroutine */ + int addrOutA; /* Address of the output-A subroutine */ + int addrOutB = 0; /* Address of the output-B subroutine */ + int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofB; /* Address of the select-B-exhausted subroutine */ + int addrAltB; /* Address of the AB subroutine */ + int regLimitA; /* Limit register for select-A */ + int regLimitB; /* Limit register for select-A */ + int regPrev; /* A range of registers to hold previous output */ + int savedLimit; /* Saved value of p->iLimit */ + int savedOffset; /* Saved value of p->iOffset */ + int labelCmpr; /* Label for the start of the merge algorithm */ + int labelEnd; /* Label for the end of the overall SELECT stmt */ + int j1; /* Jump instructions that get retargetted */ + int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ + KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ + KeyInfo *pKeyMerge; /* Comparison information for merging rows */ + sqlite3 *db; /* Database connection */ + ExprList *pOrderBy; /* The ORDER BY clause */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + int *aPermute; /* Mapping from ORDER BY terms to result set columns */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif - assert( sqlite3BtreeHoldsAllMutexes(db) ); - assert( sqlite3_mutex_held(db->mutex) ); + assert( p->pOrderBy!=0 ); + assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ + db = pParse->db; + v = pParse->pVdbe; + assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ + labelEnd = sqlite3VdbeMakeLabel(v); + labelCmpr = sqlite3VdbeMakeLabel(v); - if( p ){ - sqlite3ExpirePreparedStatements(db); - do { - VTable *pNext = p->pNext; - sqlite3VtabUnlock(p); - p = pNext; - }while( p ); - } -} -/* -** Clear any and all virtual-table information from the Table record. -** This routine is called, for example, just before deleting the Table -** record. -** -** Since it is a virtual-table, the Table structure contains a pointer -** to the head of a linked list of VTable structures. Each VTable -** structure is associated with a single sqlite3* user of the schema. -** The reference count of the VTable structure associated with database -** connection db is decremented immediately (which may lead to the -** structure being xDisconnected and free). Any other VTable structures -** in the list are moved to the sqlite3.pDisconnect list of the associated -** database connection. -*/ -SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ - if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); - if( p->azModuleArg ){ - int i; - for(i=0; inModuleArg; i++){ - sqlite3DbFree(db, p->azModuleArg[i]); + /* Patch up the ORDER BY clause + */ + op = p->op; + pPrior = p->pPrior; + assert( pPrior->pOrderBy==0 ); + pOrderBy = p->pOrderBy; + assert( pOrderBy ); + nOrderBy = pOrderBy->nExpr; + + /* For operators other than UNION ALL we have to make sure that + ** the ORDER BY clause covers every term of the result set. Add + ** terms to the ORDER BY clause as necessary. + */ + if( op!=TK_ALL ){ + for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ + struct ExprList_item *pItem; + for(j=0, pItem=pOrderBy->a; jiCol>0 ); + if( pItem->iCol==i ) break; + } + if( j==nOrderBy ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->flags |= EP_IntValue; + pNew->u.iValue = i; + pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); + pOrderBy->a[nOrderBy++].iCol = (u16)i; + } } - sqlite3DbFree(db, p->azModuleArg); } -} -/* -** Add a new module argument to pTable->azModuleArg[]. -** The string is not copied - the pointer is stored. The -** string will be freed automatically when the table is -** deleted. -*/ -static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int i = pTable->nModuleArg++; - int nBytes = sizeof(char *)*(1+pTable->nModuleArg); - char **azModuleArg; - azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); - if( azModuleArg==0 ){ - int j; - for(j=0; jazModuleArg[j]); + /* Compute the comparison permutation and keyinfo that is used with + ** the permutation used to determine if the next + ** row of results comes from selectA or selectB. Also add explicit + ** collations to the ORDER BY clause terms so that when the subqueries + ** to the right and the left are evaluated, they use the correct + ** collation. + */ + aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); + if( aPermute ){ + struct ExprList_item *pItem; + for(i=0, pItem=pOrderBy->a; iiCol>0 && pItem->iCol<=p->pEList->nExpr ); + aPermute[i] = pItem->iCol - 1; + } + pKeyMerge = + sqlite3DbMallocRaw(db, sizeof(*pKeyMerge)+nOrderBy*(sizeof(CollSeq*)+1)); + if( pKeyMerge ){ + pKeyMerge->aSortOrder = (u8*)&pKeyMerge->aColl[nOrderBy]; + pKeyMerge->nField = (u16)nOrderBy; + pKeyMerge->enc = ENC(db); + for(i=0; ia[i].pExpr; + if( pTerm->flags & EP_ExpCollate ){ + pColl = pTerm->pColl; + }else{ + pColl = multiSelectCollSeq(pParse, p, aPermute[i]); + pTerm->flags |= EP_ExpCollate; + pTerm->pColl = pColl; + } + pKeyMerge->aColl[i] = pColl; + pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; + } } - sqlite3DbFree(db, zArg); - sqlite3DbFree(db, pTable->azModuleArg); - pTable->nModuleArg = 0; }else{ - azModuleArg[i] = zArg; - azModuleArg[i+1] = 0; + pKeyMerge = 0; } - pTable->azModuleArg = azModuleArg; -} - -/* -** The parser calls this routine when it first sees a CREATE VIRTUAL TABLE -** statement. The module name has been parsed, but the optional list -** of parameters that follow the module name are still pending. -*/ -SQLITE_PRIVATE void sqlite3VtabBeginParse( - Parse *pParse, /* Parsing context */ - Token *pName1, /* Name of new table, or database name */ - Token *pName2, /* Name of new table or NULL */ - Token *pModuleName /* Name of the module for the virtual table */ -){ - int iDb; /* The database the table is being created in */ - Table *pTable; /* The new virtual table */ - sqlite3 *db; /* Database connection */ - - sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, 0); - pTable = pParse->pNewTable; - if( pTable==0 ) return; - assert( 0==pTable->pIndex ); - - db = pParse->db; - iDb = sqlite3SchemaToIndex(db, pTable->pSchema); - assert( iDb>=0 ); - pTable->tabFlags |= TF_Virtual; - pTable->nModuleArg = 0; - addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, db->aDb[iDb].zName)); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + /* Reattach the ORDER BY clause to the query. + */ + p->pOrderBy = pOrderBy; + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); -#ifndef SQLITE_OMIT_AUTHORIZATION - /* Creating a virtual table invokes the authorization callback twice. - ** The first invocation, to obtain permission to INSERT a row into the - ** sqlite_master table, has already been made by sqlite3StartTable(). - ** The second call, to obtain permission to create the table, is made now. + /* Allocate a range of temporary registers and the KeyInfo needed + ** for the logic that removes duplicate result rows when the + ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). */ - if( pTable->azModuleArg ){ - sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, - pTable->azModuleArg[0], pParse->db->aDb[iDb].zName); + if( op==TK_ALL ){ + regPrev = 0; + }else{ + int nExpr = p->pEList->nExpr; + assert( nOrderBy>=nExpr || db->mallocFailed ); + regPrev = sqlite3GetTempRange(pParse, nExpr+1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); + pKeyDup = sqlite3DbMallocZero(db, + sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); + if( pKeyDup ){ + pKeyDup->aSortOrder = (u8*)&pKeyDup->aColl[nExpr]; + pKeyDup->nField = (u16)nExpr; + pKeyDup->enc = ENC(db); + for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); + pKeyDup->aSortOrder[i] = 0; + } + } + } + + /* Separate the left and the right query from one another + */ + p->pPrior = 0; + sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); + if( pPrior->pPrior==0 ){ + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); } -#endif -} -/* -** This routine takes the module argument that has been accumulating -** in pParse->zArg[] and appends it to the list of arguments on the -** virtual table currently under construction in pParse->pTable. -*/ -static void addArgumentToVtab(Parse *pParse){ - if( pParse->sArg.z && ALWAYS(pParse->pNewTable) ){ - const char *z = (const char*)pParse->sArg.z; - int n = pParse->sArg.n; - sqlite3 *db = pParse->db; - addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + /* Compute the limit registers */ + computeLimitRegisters(pParse, p, labelEnd); + if( p->iLimit && op==TK_ALL ){ + regLimitA = ++pParse->nMem; + regLimitB = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, + regLimitA); + sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB); + }else{ + regLimitA = regLimitB = 0; } -} + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = 0; + sqlite3ExprDelete(db, p->pOffset); + p->pOffset = 0; -/* -** The parser calls this routine after the CREATE VIRTUAL TABLE statement -** has been completely parsed. -*/ -SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ - Table *pTab = pParse->pNewTable; /* The table being constructed */ - sqlite3 *db = pParse->db; /* The database connection */ + regAddrA = ++pParse->nMem; + regEofA = ++pParse->nMem; + regAddrB = ++pParse->nMem; + regEofB = ++pParse->nMem; + regOutA = ++pParse->nMem; + regOutB = ++pParse->nMem; + sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); + sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); - if( pTab==0 ) return; - addArgumentToVtab(pParse); - pParse->sArg.z = 0; - if( pTab->nModuleArg<1 ) return; - - /* If the CREATE VIRTUAL TABLE statement is being entered for the - ** first time (in other words if the virtual table is actually being - ** created now instead of just being read out of sqlite_master) then - ** do additional initialization work and store the statement text - ** in the sqlite_master table. + /* Jump past the various subroutines and coroutines to the main + ** merge loop */ - if( !db->init.busy ){ - char *zStmt; - char *zWhere; - int iDb; - Vdbe *v; - - /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ - if( pEnd ){ - pParse->sNameToken.n = (int)(pEnd->z - pParse->sNameToken.z) + pEnd->n; - } - zStmt = sqlite3MPrintf(db, "CREATE VIRTUAL TABLE %T", &pParse->sNameToken); + j1 = sqlite3VdbeAddOp0(v, OP_Goto); + addrSelectA = sqlite3VdbeCurrentAddr(v); - /* A slot for the record has already been allocated in the - ** SQLITE_MASTER table. We just need to update that slot with all - ** the information we've collected. - ** - ** The VM register number pParse->regRowid holds the rowid of an - ** entry in the sqlite_master table tht was created for this vtab - ** by sqlite3StartTable(). - */ - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - sqlite3NestedParse(pParse, - "UPDATE %Q.%s " - "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " - "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pTab->zName, - pTab->zName, - zStmt, - pParse->regRowid - ); - sqlite3DbFree(db, zStmt); - v = sqlite3GetVdbe(pParse); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); - zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 1, 0, zWhere, P4_DYNAMIC); - sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, - pTab->zName, sqlite3Strlen30(pTab->zName) + 1); - } + /* Generate a coroutine to evaluate the SELECT statement to the + ** left of the compound operator - the "A" select. + */ + VdbeNoopComment((v, "Begin coroutine for left SELECT")); + pPrior->iLimit = regLimitA; + explainSetInteger(iSub1, pParse->iNextSelectId); + sqlite3Select(pParse, pPrior, &destA); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + VdbeNoopComment((v, "End coroutine for left SELECT")); - /* If we are rereading the sqlite_master table create the in-memory - ** record of the table. The xConnect() method is not called until - ** the first time the virtual table is used in an SQL statement. This - ** allows a schema that contains virtual tables to be loaded before - ** the required virtual table implementations are registered. */ - else { - Table *pOld; - Schema *pSchema = pTab->pSchema; - const char *zName = pTab->zName; - int nName = sqlite3Strlen30(zName); - pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); - if( pOld ){ - db->mallocFailed = 1; - assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ - return; - } - pParse->pNewTable = 0; - } -} - -/* -** The parser calls this routine when it sees the first token -** of an argument to the module name in a CREATE VIRTUAL TABLE statement. -*/ -SQLITE_PRIVATE void sqlite3VtabArgInit(Parse *pParse){ - addArgumentToVtab(pParse); - pParse->sArg.z = 0; - pParse->sArg.n = 0; -} - -/* -** The parser calls this routine for each token after the first token -** in an argument to the module name in a CREATE VIRTUAL TABLE statement. -*/ -SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ - Token *pArg = &pParse->sArg; - if( pArg->z==0 ){ - pArg->z = p->z; - pArg->n = p->n; - }else{ - assert(pArg->z < p->z); - pArg->n = (int)(&p->z[p->n] - pArg->z); - } -} - -/* -** Invoke a virtual table constructor (either xCreate or xConnect). The -** pointer to the function to invoke is passed as the fourth parameter -** to this procedure. -*/ -static int vtabCallConstructor( - sqlite3 *db, - Table *pTab, - Module *pMod, - int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), - char **pzErr -){ - VTable *pVTable; - int rc; - const char *const*azArg = (const char *const*)pTab->azModuleArg; - int nArg = pTab->nModuleArg; - char *zErr = 0; - char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); - - if( !zModuleName ){ - return SQLITE_NOMEM; - } + /* Generate a coroutine to evaluate the SELECT statement on + ** the right - the "B" select + */ + addrSelectB = sqlite3VdbeCurrentAddr(v); + VdbeNoopComment((v, "Begin coroutine for right SELECT")); + savedLimit = p->iLimit; + savedOffset = p->iOffset; + p->iLimit = regLimitB; + p->iOffset = 0; + explainSetInteger(iSub2, pParse->iNextSelectId); + sqlite3Select(pParse, p, &destB); + p->iLimit = savedLimit; + p->iOffset = savedOffset; + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofB); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + VdbeNoopComment((v, "End coroutine for right SELECT")); - pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); - if( !pVTable ){ - sqlite3DbFree(db, zModuleName); - return SQLITE_NOMEM; + /* Generate a subroutine that outputs the current row of the A + ** select as the next output row of the compound select. + */ + VdbeNoopComment((v, "Output routine for A")); + addrOutA = generateOutputSubroutine(pParse, + p, &destA, pDest, regOutA, + regPrev, pKeyDup, P4_KEYINFO_HANDOFF, labelEnd); + + /* Generate a subroutine that outputs the current row of the B + ** select as the next output row of the compound select. + */ + if( op==TK_ALL || op==TK_UNION ){ + VdbeNoopComment((v, "Output routine for B")); + addrOutB = generateOutputSubroutine(pParse, + p, &destB, pDest, regOutB, + regPrev, pKeyDup, P4_KEYINFO_STATIC, labelEnd); } - pVTable->db = db; - pVTable->pMod = pMod; - - assert( !db->pVTab ); - assert( xConstruct ); - db->pVTab = pTab; - - /* Invoke the virtual table constructor */ - rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - - if( SQLITE_OK!=rc ){ - if( zErr==0 ){ - *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); - }else { - *pzErr = sqlite3MPrintf(db, "%s", zErr); - sqlite3_free(zErr); - } - sqlite3DbFree(db, pVTable); - }else if( ALWAYS(pVTable->pVtab) ){ - /* Justification of ALWAYS(): A correct vtab constructor must allocate - ** the sqlite3_vtab object if successful. */ - pVTable->pVtab->pModule = pMod->pModule; - pVTable->nRef = 1; - if( db->pVTab ){ - const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); - sqlite3VtabUnlock(pVTable); - rc = SQLITE_ERROR; - }else{ - int iCol; - /* If everything went according to plan, link the new VTable structure - ** into the linked list headed by pTab->pVTable. Then loop through the - ** columns of the table to see if any of them contain the token "hidden". - ** If so, set the Column.isHidden flag and remove the token from - ** the type string. */ - pVTable->pNext = pTab->pVTable; - pTab->pVTable = pVTable; - for(iCol=0; iColnCol; iCol++){ - char *zType = pTab->aCol[iCol].zType; - int nType; - int i = 0; - if( !zType ) continue; - nType = sqlite3Strlen30(zType); - if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){ - for(i=0; i0 ){ - assert(zType[i-1]==' '); - zType[i-1] = '\0'; - } - pTab->aCol[iCol].isHidden = 1; - } - } - } + /* Generate a subroutine to run when the results from select A + ** are exhausted and only data in select B remains. + */ + VdbeNoopComment((v, "eof-A subroutine")); + if( op==TK_EXCEPT || op==TK_INTERSECT ){ + addrEofA = sqlite3VdbeAddOp2(v, OP_Goto, 0, labelEnd); + }else{ + addrEofA = sqlite3VdbeAddOp2(v, OP_If, regEofB, labelEnd); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + p->nSelectRow += pPrior->nSelectRow; } - sqlite3DbFree(db, zModuleName); - db->pVTab = 0; - return rc; -} - -/* -** This function is invoked by the parser to call the xConnect() method -** of the virtual table pTab. If an error occurs, an error code is returned -** and an error left in pParse. -** -** This call is a no-op if table pTab is not a virtual table. -*/ -SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ - sqlite3 *db = pParse->db; - const char *zMod; - Module *pMod; - int rc; - - assert( pTab ); - if( (pTab->tabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){ - return SQLITE_OK; + /* Generate a subroutine to run when the results from select B + ** are exhausted and only data in select A remains. + */ + if( op==TK_INTERSECT ){ + addrEofB = addrEofA; + if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; + }else{ + VdbeNoopComment((v, "eof-B subroutine")); + addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); } - /* Locate the required virtual table module */ - zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + /* Generate code to handle the case of AazModuleArg[0]; - sqlite3ErrorMsg(pParse, "no such module: %s", zModule); - rc = SQLITE_ERROR; + /* Generate code to handle the case of A==B + */ + if( op==TK_ALL ){ + addrAeqB = addrAltB; + }else if( op==TK_INTERSECT ){ + addrAeqB = addrAltB; + addrAltB++; }else{ - char *zErr = 0; - rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); - if( rc!=SQLITE_OK ){ - sqlite3ErrorMsg(pParse, "%s", zErr); - } - sqlite3DbFree(db, zErr); + VdbeNoopComment((v, "A-eq-B subroutine")); + addrAeqB = + sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); } - return rc; -} - -/* -** Add the virtual table pVTab to the array sqlite3.aVTrans[]. -*/ -static int addToVTrans(sqlite3 *db, VTable *pVTab){ - const int ARRAY_INCR = 5; - - /* Grow the sqlite3.aVTrans array if required */ - if( (db->nVTrans%ARRAY_INCR)==0 ){ - VTable **aVTrans; - int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); - aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); - if( !aVTrans ){ - return SQLITE_NOMEM; - } - memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); - db->aVTrans = aVTrans; + /* Generate code to handle the case of A>B + */ + VdbeNoopComment((v, "A-gt-B subroutine")); + addrAgtB = sqlite3VdbeCurrentAddr(v); + if( op==TK_ALL || op==TK_UNION ){ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } + sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); - /* Add pVtab to the end of sqlite3.aVTrans */ - db->aVTrans[db->nVTrans++] = pVTab; - sqlite3VtabLock(pVTab); - return SQLITE_OK; -} + /* This code runs once to initialize everything. + */ + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofA); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofB); + sqlite3VdbeAddOp2(v, OP_Gosub, regAddrA, addrSelectA); + sqlite3VdbeAddOp2(v, OP_Gosub, regAddrB, addrSelectB); + sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); -/* -** This function is invoked by the vdbe to call the xCreate method -** of the virtual table named zTab in database iDb. -** -** If an error occurs, *pzErr is set to point an an English language -** description of the error and an SQLITE_XXX error code is returned. -** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. -*/ -SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ - int rc = SQLITE_OK; - Table *pTab; - Module *pMod; - const char *zMod; + /* Implement the main merge loop + */ + sqlite3VdbeResolveLabel(v, labelCmpr); + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, + (char*)pKeyMerge, P4_KEYINFO_HANDOFF); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); - assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); + /* Release temporary registers + */ + if( regPrev ){ + sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); + } - /* Locate the required virtual table module */ - zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + /* Jump to the this point in order to terminate the query. + */ + sqlite3VdbeResolveLabel(v, labelEnd); - /* If the module has been registered and includes a Create method, - ** invoke it now. If the module has not been registered, return an - ** error. Otherwise, do nothing. + /* Set the number of output columns */ - if( !pMod ){ - *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); - rc = SQLITE_ERROR; - }else{ - rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xCreate, pzErr); + if( pDest->eDest==SRT_Output ){ + Select *pFirst = pPrior; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); } - /* Justification of ALWAYS(): The xConstructor method is required to - ** create a valid sqlite3_vtab if it returns SQLITE_OK. */ - if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ - rc = addToVTrans(db, sqlite3GetVTable(db, pTab)); + /* Reassembly the compound query so that it will be freed correctly + ** by the calling function */ + if( p->pPrior ){ + sqlite3SelectDelete(db, p->pPrior); } + p->pPrior = pPrior; - return rc; + /*** TBD: Insert subroutine calls to close cursors on incomplete + **** subqueries ****/ + explainComposite(pParse, p->op, iSub1, iSub2, 0); + return SQLITE_OK; } +#endif + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* Forward Declarations */ +static void substExprList(sqlite3*, ExprList*, int, ExprList*); +static void substSelect(sqlite3*, Select *, int, ExprList *); /* -** This function is used to set the schema of a virtual table. It is only -** valid to call this function from within the xCreate() or xConnect() of a -** virtual table module. +** Scan through the expression pExpr. Replace every reference to +** a column in table number iTable with a copy of the iColumn-th +** entry in pEList. (But leave references to the ROWID column +** unchanged.) +** +** This routine is part of the flattening procedure. A subquery +** whose result set is defined by pEList appears as entry in the +** FROM clause of a SELECT such that the VDBE cursor assigned to that +** FORM clause entry is iTable. This routine make the necessary +** changes to pExpr so that it refers directly to the source table +** of the subquery rather the result set of the subquery. */ -SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ - Parse *pParse; - - int rc = SQLITE_OK; - Table *pTab; - char *zErr = 0; - - sqlite3_mutex_enter(db->mutex); - pTab = db->pVTab; - if( !pTab ){ - sqlite3Error(db, SQLITE_MISUSE, 0); - sqlite3_mutex_leave(db->mutex); - return SQLITE_MISUSE_BKPT; - } - assert( (pTab->tabFlags & TF_Virtual)!=0 ); - - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; - }else{ - pParse->declareVtab = 1; - pParse->db = db; - pParse->nQueryLoop = 1; - - if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) - && pParse->pNewTable - && !db->mallocFailed - && !pParse->pNewTable->pSelect - && (pParse->pNewTable->tabFlags & TF_Virtual)==0 - ){ - if( !pTab->aCol ){ - pTab->aCol = pParse->pNewTable->aCol; - pTab->nCol = pParse->pNewTable->nCol; - pParse->pNewTable->nCol = 0; - pParse->pNewTable->aCol = 0; - } - db->pVTab = 0; +static Expr *substExpr( + sqlite3 *db, /* Report malloc errors to this connection */ + Expr *pExpr, /* Expr in which substitution occurs */ + int iTable, /* Table to be substituted */ + ExprList *pEList /* Substitute expressions */ +){ + if( pExpr==0 ) return 0; + if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ + if( pExpr->iColumn<0 ){ + pExpr->op = TK_NULL; }else{ - sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); - sqlite3DbFree(db, zErr); - rc = SQLITE_ERROR; + Expr *pNew; + assert( pEList!=0 && pExpr->iColumnnExpr ); + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); + if( pNew && pExpr->pColl ){ + pNew->pColl = pExpr->pColl; + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; } - pParse->declareVtab = 0; - - if( pParse->pVdbe ){ - sqlite3VdbeFinalize(pParse->pVdbe); + }else{ + pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); + pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + substSelect(db, pExpr->x.pSelect, iTable, pEList); + }else{ + substExprList(db, pExpr->x.pList, iTable, pEList); } - sqlite3DeleteTable(db, pParse->pNewTable); - sqlite3StackFree(db, pParse); } - - assert( (rc&0xff)==rc ); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return pExpr; } - -/* -** This function is invoked by the vdbe to call the xDestroy method -** of the virtual table named zTab in database iDb. This occurs -** when a DROP TABLE is mentioned. -** -** This call is a no-op if zTab is not a virtual table. -*/ -SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ - int rc = SQLITE_OK; - Table *pTab; - - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); - if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ - VTable *p = vtabDisconnectAll(db, pTab); - - assert( rc==SQLITE_OK ); - rc = p->pMod->pModule->xDestroy(p->pVtab); - - /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ - if( rc==SQLITE_OK ){ - assert( pTab->pVTable==p && p->pNext==0 ); - p->pVtab = 0; - pTab->pVTable = 0; - sqlite3VtabUnlock(p); +static void substExprList( + sqlite3 *db, /* Report malloc errors here */ + ExprList *pList, /* List to scan and in which to make substitutes */ + int iTable, /* Table to be substituted */ + ExprList *pEList /* Substitute values */ +){ + int i; + if( pList==0 ) return; + for(i=0; inExpr; i++){ + pList->a[i].pExpr = substExpr(db, pList->a[i].pExpr, iTable, pEList); + } +} +static void substSelect( + sqlite3 *db, /* Report malloc errors here */ + Select *p, /* SELECT statement in which to make substitutions */ + int iTable, /* Table to be replaced */ + ExprList *pEList /* Substitute values */ +){ + SrcList *pSrc; + struct SrcList_item *pItem; + int i; + if( !p ) return; + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + substSelect(db, p->pPrior, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ + if( ALWAYS(pSrc) ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(db, pItem->pSelect, iTable, pEList); } } - - return rc; } +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* -** This function invokes either the xRollback or xCommit method -** of each of the virtual tables in the sqlite3.aVTrans array. The method -** called is identified by the second argument, "offset", which is -** the offset of the method to call in the sqlite3_module structure. +** This routine attempts to flatten subqueries in order to speed +** execution. It returns 1 if it makes changes and 0 if no flattening +** occurs. ** -** The array is cleared after invoking the callbacks. -*/ -static void callFinaliser(sqlite3 *db, int offset){ - int i; - if( db->aVTrans ){ - for(i=0; inVTrans; i++){ - VTable *pVTab = db->aVTrans[i]; - sqlite3_vtab *p = pVTab->pVtab; - if( p ){ - int (*x)(sqlite3_vtab *); - x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset); - if( x ) x(p); - } - sqlite3VtabUnlock(pVTab); - } - sqlite3DbFree(db, db->aVTrans); - db->nVTrans = 0; - db->aVTrans = 0; - } -} - -/* -** Invoke the xSync method of all virtual tables in the sqlite3.aVTrans -** array. Return the error code for the first error that occurs, or -** SQLITE_OK if all xSync operations are successful. +** To understand the concept of flattening, consider the following +** query: ** -** Set *pzErrmsg to point to a buffer that should be released using -** sqlite3DbFree() containing an error message, if one is available. -*/ -SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ - int i; - int rc = SQLITE_OK; - VTable **aVTrans = db->aVTrans; - - db->aVTrans = 0; - for(i=0; rc==SQLITE_OK && inVTrans; i++){ - int (*x)(sqlite3_vtab *); - sqlite3_vtab *pVtab = aVTrans[i]->pVtab; - if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ - rc = x(pVtab); - sqlite3DbFree(db, *pzErrmsg); - *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg); - sqlite3_free(pVtab->zErrMsg); - } - } - db->aVTrans = aVTrans; - return rc; -} - -/* -** Invoke the xRollback method of all virtual tables in the -** sqlite3.aVTrans array. Then clear the array itself. -*/ -SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db){ - callFinaliser(db, offsetof(sqlite3_module,xRollback)); - return SQLITE_OK; -} - -/* -** Invoke the xCommit method of all virtual tables in the -** sqlite3.aVTrans array. Then clear the array itself. -*/ -SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db){ - callFinaliser(db, offsetof(sqlite3_module,xCommit)); - return SQLITE_OK; -} - -/* -** If the virtual table pVtab supports the transaction interface -** (xBegin/xRollback/xCommit and optionally xSync) and a transaction is -** not currently open, invoke the xBegin method now. +** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5 ** -** If the xBegin call is successful, place the sqlite3_vtab pointer -** in the sqlite3.aVTrans array. +** The default way of implementing this query is to execute the +** subquery first and store the results in a temporary table, then +** run the outer query on that temporary table. This requires two +** passes over the data. Furthermore, because the temporary table +** has no indices, the WHERE clause on the outer query cannot be +** optimized. +** +** This routine attempts to rewrite queries such as the above into +** a single flat select, like this: +** +** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 +** +** The code generated for this simpification gives the same result +** but only has to scan the data once. And because indices might +** exist on the table t1, a complete scan of the data might be +** avoided. +** +** Flattening is only attempted if all of the following are true: +** +** (1) The subquery and the outer query do not both use aggregates. +** +** (2) The subquery is not an aggregate or the outer query is not a join. +** +** (3) The subquery is not the right operand of a left outer join +** (Originally ticket #306. Strengthened by ticket #3300) +** +** (4) The subquery is not DISTINCT. +** +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. +** +** (6) The subquery does not use aggregates or the outer query is not +** DISTINCT. +** +** (7) The subquery has a FROM clause. +** +** (8) The subquery does not use LIMIT or the outer query is not a join. +** +** (9) The subquery does not use LIMIT or the outer query does not use +** aggregates. +** +** (10) The subquery does not use aggregates or the outer query does not +** use LIMIT. +** +** (11) The subquery and the outer query do not both have ORDER BY clauses. +** +** (**) Not implemented. Subsumed into restriction (3). Was previously +** a separate restriction deriving from ticket #350. +** +** (13) The subquery and outer query do not both use LIMIT. +** +** (14) The subquery does not use OFFSET. +** +** (15) The outer query is not part of a compound select or the +** subquery does not have a LIMIT clause. +** (See ticket #2339 and ticket [02a8e81d44]). +** +** (16) The outer query is not an aggregate or the subquery does +** not contain ORDER BY. (Ticket #2942) This used to not matter +** until we introduced the group_concat() function. +** +** (17) The sub-query is not a compound select, or it is a UNION ALL +** compound clause made up entirely of non-aggregate queries, and +** the parent query: +** +** * is not itself part of a compound select, +** * is not an aggregate or DISTINCT query, and +** * has no other tables or sub-selects in the FROM clause. +** +** The parent and sub-query may contain WHERE clauses. Subject to +** rules (11), (13) and (14), they may also contain ORDER BY, +** LIMIT and OFFSET clauses. +** +** (18) If the sub-query is a compound select, then all terms of the +** ORDER by clause of the parent must be simple references to +** columns of the sub-query. +** +** (19) The subquery does not use LIMIT or the outer query does not +** have a WHERE clause. +** +** (20) If the sub-query is a compound select, then it must not use +** an ORDER BY clause. Ticket #3773. We could relax this constraint +** somewhat by saying that the terms of the ORDER BY clause must +** appear as unmodified result columns in the outer query. But +** have other optimizations in mind to deal with that case. +** +** (21) The subquery does not use LIMIT or the outer query is not +** DISTINCT. (See ticket [752e1646fc]). +** +** In this routine, the "p" parameter is a pointer to the outer query. +** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query +** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. +** +** If flattening is not attempted, this routine is a no-op and returns 0. +** If flattening is attempted this routine returns 1. +** +** All of the expression analysis must occur on both the outer query and +** the subquery before this routine runs. */ -SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ - int rc = SQLITE_OK; - const sqlite3_module *pModule; +static int flattenSubquery( + Parse *pParse, /* Parsing context */ + Select *p, /* The parent or outer SELECT statement */ + int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ + int isAgg, /* True if outer SELECT uses aggregate functions */ + int subqueryIsAgg /* True if the subquery uses aggregate functions */ +){ + const char *zSavedAuthContext = pParse->zAuthContext; + Select *pParent; + Select *pSub; /* The inner query or "subquery" */ + Select *pSub1; /* Pointer to the rightmost select in sub-query */ + SrcList *pSrc; /* The FROM clause of the outer query */ + SrcList *pSubSrc; /* The FROM clause of the subquery */ + ExprList *pList; /* The result set of the outer query */ + int iParent; /* VDBE cursor number of the pSub result set temp table */ + int i; /* Loop counter */ + Expr *pWhere; /* The WHERE clause */ + struct SrcList_item *pSubitem; /* The subquery */ + sqlite3 *db = pParse->db; - /* Special case: If db->aVTrans is NULL and db->nVTrans is greater - ** than zero, then this function is being called from within a - ** virtual module xSync() callback. It is illegal to write to - ** virtual module tables in this case, so return SQLITE_LOCKED. + /* Check to see if flattening is permitted. Return 0 if not. */ - if( sqlite3VtabInSync(db) ){ - return SQLITE_LOCKED; + assert( p!=0 ); + assert( p->pPrior==0 ); /* Unable to flatten compound queries */ + if( db->flags & SQLITE_QueryFlattener ) return 0; + pSrc = p->pSrc; + assert( pSrc && iFrom>=0 && iFromnSrc ); + pSubitem = &pSrc->a[iFrom]; + iParent = pSubitem->iCursor; + pSub = pSubitem->pSelect; + assert( pSub!=0 ); + if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ + if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ + pSubSrc = pSub->pSrc; + assert( pSubSrc ); + /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, + ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET + ** because they could be computed at compile-time. But when LIMIT and OFFSET + ** became arbitrary expressions, we were forced to add restrictions (13) + ** and (14). */ + if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ + if( pSub->pOffset ) return 0; /* Restriction (14) */ + if( p->pRightmost && pSub->pLimit ){ + return 0; /* Restriction (15) */ + } + if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ + } + if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ + return 0; /* Restriction (6) */ + } + if( p->pOrderBy && pSub->pOrderBy ){ + return 0; /* Restriction (11) */ + } + if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ + if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ + if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ + return 0; /* Restriction (21) */ } - if( !pVTab ){ - return SQLITE_OK; - } - pModule = pVTab->pVtab->pModule; - - if( pModule->xBegin ){ - int i; + /* OBSOLETE COMMENT 1: + ** Restriction 3: If the subquery is a join, make sure the subquery is + ** not used as the right operand of an outer join. Examples of why this + ** is not allowed: + ** + ** t1 LEFT OUTER JOIN (t2 JOIN t3) + ** + ** If we flatten the above, we would get + ** + ** (t1 LEFT OUTER JOIN t2) JOIN t3 + ** + ** which is not at all the same thing. + ** + ** OBSOLETE COMMENT 2: + ** Restriction 12: If the subquery is the right operand of a left outer + ** join, make sure the subquery has no WHERE clause. + ** An examples of why this is not allowed: + ** + ** t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0) + ** + ** If we flatten the above, we would get + ** + ** (t1 LEFT OUTER JOIN t2) WHERE t2.x>0 + ** + ** But the t2.x>0 test will always fail on a NULL row of t2, which + ** effectively converts the OUTER JOIN into an INNER JOIN. + ** + ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE: + ** Ticket #3300 shows that flattening the right term of a LEFT JOIN + ** is fraught with danger. Best to avoid the whole thing. If the + ** subquery is the right term of a LEFT JOIN, then do not flatten. + */ + if( (pSubitem->jointype & JT_OUTER)!=0 ){ + return 0; + } - /* If pVtab is already in the aVTrans array, return early */ - for(i=0; inVTrans; i++){ - if( db->aVTrans[i]==pVTab ){ - return SQLITE_OK; + /* Restriction 17: If the sub-query is a compound SELECT, then it must + ** use only the UNION ALL operator. And none of the simple select queries + ** that make up the compound SELECT are allowed to be aggregate or distinct + ** queries. + */ + if( pSub->pPrior ){ + if( pSub->pOrderBy ){ + return 0; /* Restriction 20 */ + } + if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){ + return 0; + } + for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 + || (pSub1->pPrior && pSub1->op!=TK_ALL) + || NEVER(pSub1->pSrc==0) || pSub1->pSrc->nSrc!=1 + ){ + return 0; } } - /* Invoke the xBegin method */ - rc = pModule->xBegin(pVTab->pVtab); - if( rc==SQLITE_OK ){ - rc = addToVTrans(db, pVTab); + /* Restriction 18. */ + if( p->pOrderBy ){ + int ii; + for(ii=0; iipOrderBy->nExpr; ii++){ + if( p->pOrderBy->a[ii].iCol==0 ) return 0; + } } } - return rc; -} -/* -** The first parameter (pDef) is a function implementation. The -** second parameter (pExpr) is the first argument to this function. -** If pExpr is a column in a virtual table, then let the virtual -** table implementation have an opportunity to overload the function. -** -** This routine is used to allow virtual table implementations to -** overload MATCH, LIKE, GLOB, and REGEXP operators. -** -** Return either the pDef argument (indicating no change) or a -** new FuncDef structure that is marked as ephemeral using the -** SQLITE_FUNC_EPHEM flag. -*/ -SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( - sqlite3 *db, /* Database connection for reporting malloc problems */ - FuncDef *pDef, /* Function to possibly overload */ - int nArg, /* Number of arguments to the function */ - Expr *pExpr /* First argument to the function */ -){ - Table *pTab; - sqlite3_vtab *pVtab; - sqlite3_module *pMod; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; - void *pArg = 0; - FuncDef *pNew; - int rc = 0; - char *zLowerName; - unsigned char *z; + /***** If we reach this point, flattening is permitted. *****/ + /* Authorize the subquery */ + pParse->zAuthContext = pSubitem->zName; + sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + pParse->zAuthContext = zSavedAuthContext; - /* Check to see the left operand is a column in a virtual table */ - if( NEVER(pExpr==0) ) return pDef; - if( pExpr->op!=TK_COLUMN ) return pDef; - pTab = pExpr->pTab; - if( NEVER(pTab==0) ) return pDef; - if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef; - pVtab = sqlite3GetVTable(db, pTab)->pVtab; - assert( pVtab!=0 ); - assert( pVtab->pModule!=0 ); - pMod = (sqlite3_module *)pVtab->pModule; - if( pMod->xFindFunction==0 ) return pDef; - - /* Call the xFindFunction method on the virtual table implementation - ** to see if the implementation wants to overload this function + /* If the sub-query is a compound SELECT statement, then (by restrictions + ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** be of the form: + ** + ** SELECT FROM () + ** + ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** OFFSET clauses and joins them to the left-hand-side of the original + ** using UNION ALL operators. In this case N is the number of simple + ** select statements in the compound sub-query. + ** + ** Example: + ** + ** SELECT a+1 FROM ( + ** SELECT x FROM tab + ** UNION ALL + ** SELECT y FROM tab + ** UNION ALL + ** SELECT abs(z*2) FROM tab2 + ** ) WHERE a!=5 ORDER BY 1 + ** + ** Transformed into: + ** + ** SELECT x+1 FROM tab WHERE x+1!=5 + ** UNION ALL + ** SELECT y+1 FROM tab WHERE y+1!=5 + ** UNION ALL + ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5 + ** ORDER BY 1 + ** + ** We call this the "compound-subquery flattening". */ - zLowerName = sqlite3DbStrDup(db, pDef->zName); - if( zLowerName ){ - for(z=(unsigned char*)zLowerName; *z; z++){ - *z = sqlite3UpperToLower[*z]; + for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){ + Select *pNew; + ExprList *pOrderBy = p->pOrderBy; + Expr *pLimit = p->pLimit; + Select *pPrior = p->pPrior; + p->pOrderBy = 0; + p->pSrc = 0; + p->pPrior = 0; + p->pLimit = 0; + pNew = sqlite3SelectDup(db, p, 0); + p->pLimit = pLimit; + p->pOrderBy = pOrderBy; + p->pSrc = pSrc; + p->op = TK_ALL; + p->pRightmost = 0; + if( pNew==0 ){ + pNew = pPrior; + }else{ + pNew->pPrior = pPrior; + pNew->pRightmost = 0; } - rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); - sqlite3DbFree(db, zLowerName); - } - if( rc==0 ){ - return pDef; + p->pPrior = pNew; + if( db->mallocFailed ) return 1; } - /* Create a new ephemeral function definition for the overloaded - ** function */ - pNew = sqlite3DbMallocZero(db, sizeof(*pNew) - + sqlite3Strlen30(pDef->zName) + 1); - if( pNew==0 ){ - return pDef; - } - *pNew = *pDef; - pNew->zName = (char *)&pNew[1]; - memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); - pNew->xFunc = xFunc; - pNew->pUserData = pArg; - pNew->flags |= SQLITE_FUNC_EPHEM; - return pNew; -} + /* Begin flattening the iFrom-th entry of the FROM clause + ** in the outer query. + */ + pSub = pSub1 = pSubitem->pSelect; -/* -** Make sure virtual table pTab is contained in the pParse->apVirtualLock[] -** array so that an OP_VBegin will get generated for it. Add pTab to the -** array if it is missing. If pTab is already in the array, this routine -** is a no-op. -*/ -SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - int i, n; - Table **apVtabLock; + /* Delete the transient table structure associated with the + ** subquery + */ + sqlite3DbFree(db, pSubitem->zDatabase); + sqlite3DbFree(db, pSubitem->zName); + sqlite3DbFree(db, pSubitem->zAlias); + pSubitem->zDatabase = 0; + pSubitem->zName = 0; + pSubitem->zAlias = 0; + pSubitem->pSelect = 0; - assert( IsVirtual(pTab) ); - for(i=0; inVtabLock; i++){ - if( pTab==pToplevel->apVtabLock[i] ) return; - } - n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); - apVtabLock = sqlite3_realloc(pToplevel->apVtabLock, n); - if( apVtabLock ){ - pToplevel->apVtabLock = apVtabLock; - pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; - }else{ - pToplevel->db->mallocFailed = 1; + /* Defer deleting the Table object associated with the + ** subquery until code generation is + ** complete, since there may still exist Expr.pTab entries that + ** refer to the subquery even after flattening. Ticket #3346. + ** + ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + */ + if( ALWAYS(pSubitem->pTab!=0) ){ + Table *pTabToDel = pSubitem->pTab; + if( pTabToDel->nRef==1 ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pTabToDel->pNextZombie = pToplevel->pZombieTab; + pToplevel->pZombieTab = pTabToDel; + }else{ + pTabToDel->nRef--; + } + pSubitem->pTab = 0; } -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ + /* The following loop runs once for each term in a compound-subquery + ** flattening (as described above). If we are doing a different kind + ** of flattening - a flattening other than a compound-subquery flattening - + ** then this loop only runs once. + ** + ** This loop moves all of the FROM elements of the subquery into the + ** the FROM clause of the outer query. Before doing this, remember + ** the cursor number for the original outer query FROM element in + ** iParent. The iParent cursor will never be used. Subsequent code + ** will scan expressions looking for iParent references and replace + ** those references with expressions that resolve to the subquery FROM + ** elements we are now copying in. + */ + for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ + int nSubSrc; + u8 jointype = 0; + pSubSrc = pSub->pSrc; /* FROM clause of subquery */ + nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ + pSrc = pParent->pSrc; /* FROM clause of the outer query */ -/************** End of vtab.c ************************************************/ -/************** Begin file where.c *******************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. This module is responsible for -** generating the code that loops through a table looking for applicable -** rows. Indices are selected and used to speed the search when doing -** so is applicable. Because this module is responsible for selecting -** indices, you might also think of this module as the "query optimizer". -*/ - -/* -** Trace output macros -*/ -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3WhereTrace = 0; -#endif -#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) -# define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X -#else -# define WHERETRACE(X) -#endif - -/* Forward reference -*/ -typedef struct WhereClause WhereClause; -typedef struct WhereMaskSet WhereMaskSet; -typedef struct WhereOrInfo WhereOrInfo; -typedef struct WhereAndInfo WhereAndInfo; -typedef struct WhereCost WhereCost; - -/* -** The query generator uses an array of instances of this structure to -** help it analyze the subexpressions of the WHERE clause. Each WHERE -** clause subexpression is separated from the others by AND operators, -** usually, or sometimes subexpressions separated by OR. -** -** All WhereTerms are collected into a single WhereClause structure. -** The following identity holds: -** -** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm -** -** When a term is of the form: -** -** X -** -** where X is a column name and is one of certain operators, -** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the -** cursor number and column number for X. WhereTerm.eOperator records -** the using a bitmask encoding defined by WO_xxx below. The -** use of a bitmask encoding for the operator allows us to search -** quickly for terms that match any of several different operators. -** -** A WhereTerm might also be two or more subterms connected by OR: -** -** (t1.X ) OR (t1.Y ) OR .... -** -** In this second case, wtFlag as the TERM_ORINFO set and eOperator==WO_OR -** and the WhereTerm.u.pOrInfo field points to auxiliary information that -** is collected about the -** -** If a term in the WHERE clause does not match either of the two previous -** categories, then eOperator==0. The WhereTerm.pExpr field is still set -** to the original subexpression content and wtFlags is set up appropriately -** but no other fields in the WhereTerm object are meaningful. -** -** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers, -** but they do so indirectly. A single WhereMaskSet structure translates -** cursor number into bits and the translated bit is stored in the prereq -** fields. The translation is used in order to maximize the number of -** bits that will fit in a Bitmask. The VDBE cursor numbers might be -** spread out over the non-negative integers. For example, the cursor -** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45. The WhereMaskSet -** translates these sparse cursor numbers into consecutive integers -** beginning with 0 in order to make the best possible use of the available -** bits in the Bitmask. So, in the example above, the cursor numbers -** would be mapped into integers 0 through 7. -** -** The number of terms in a join is limited by the number of bits -** in prereqRight and prereqAll. The default is 64 bits, hence SQLite -** is only able to process joins with 64 or fewer tables. -*/ -typedef struct WhereTerm WhereTerm; -struct WhereTerm { - Expr *pExpr; /* Pointer to the subexpression that is this term */ - int iParent; /* Disable pWC->a[iParent] when this term disabled */ - int leftCursor; /* Cursor number of X in "X " */ - union { - int leftColumn; /* Column number of X in "X " */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ - } u; - u16 eOperator; /* A WO_xx value describing */ - u8 wtFlags; /* TERM_xxx bit flags. See below */ - u8 nChild; /* Number of children that must disable us */ - WhereClause *pWC; /* The clause this term is part of */ - Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ - Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ -}; + if( pSrc ){ + assert( pParent==p ); /* First time through the loop */ + jointype = pSubitem->jointype; + }else{ + assert( pParent!=p ); /* 2nd and subsequent times through the loop */ + pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + if( pSrc==0 ){ + assert( db->mallocFailed ); + break; + } + } -/* -** Allowed values of WhereTerm.wtFlags -*/ -#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ -#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ -#define TERM_CODED 0x04 /* This term is already coded */ -#define TERM_COPIED 0x08 /* Has a child */ -#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ -#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ -#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ + /* The subquery uses a single slot of the FROM clause of the outer + ** query. If the subquery has more than one element in its FROM clause, + ** then expand the outer query to make space for it to hold all elements + ** of the subquery. + ** + ** Example: + ** + ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; + ** + ** The outer query has 3 slots in its FROM clause. One slot of the + ** outer query (the middle slot) is used by the subquery. The next + ** block of code will expand the out query to 4 slots. The middle + ** slot is expanded to two slots in order to make space for the + ** two elements in the FROM clause of the subquery. + */ + if( nSubSrc>1 ){ + pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); + if( db->mallocFailed ){ + break; + } + } -/* -** An instance of the following structure holds all information about a -** WHERE clause. Mostly this is a container for one or more WhereTerms. -*/ -struct WhereClause { - Parse *pParse; /* The parser context */ - WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ - u8 op; /* Split operator. TK_AND or TK_OR */ - int nTerm; /* Number of terms */ - int nSlot; /* Number of entries in a[] */ - WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ -#if defined(SQLITE_SMALL_STACK) - WhereTerm aStatic[1]; /* Initial static space for a[] */ -#else - WhereTerm aStatic[8]; /* Initial static space for a[] */ -#endif -}; + /* Transfer the FROM clause terms from the subquery into the + ** outer query. + */ + for(i=0; ia[i+iFrom].pUsing); + pSrc->a[i+iFrom] = pSubSrc->a[i]; + memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); + } + pSrc->a[iFrom].jointype = jointype; + + /* Now begin substituting subquery result set expressions for + ** references to the iParent in the outer query. + ** + ** Example: + ** + ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; + ** \ \_____________ subquery __________/ / + ** \_____________________ outer query ______________________________/ + ** + ** We look at every expression in the outer query and every place we see + ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". + */ + pList = pParent->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zName==0 ){ + const char *zSpan = pList->a[i].zSpan; + if( ALWAYS(zSpan) ){ + pList->a[i].zName = sqlite3DbStrDup(db, zSpan); + } + } + } + substExprList(db, pParent->pEList, iParent, pSub->pEList); + if( isAgg ){ + substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); + pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); + } + if( pSub->pOrderBy ){ + assert( pParent->pOrderBy==0 ); + pParent->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + }else if( pParent->pOrderBy ){ + substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); + } + if( pSub->pWhere ){ + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + }else{ + pWhere = 0; + } + if( subqueryIsAgg ){ + assert( pParent->pHaving==0 ); + pParent->pHaving = pParent->pWhere; + pParent->pWhere = pWhere; + pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); + pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, + sqlite3ExprDup(db, pSub->pHaving, 0)); + assert( pParent->pGroupBy==0 ); + pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); + }else{ + pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); + pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + } + + /* The flattened query is distinct if either the inner or the + ** outer query is distinct. + */ + pParent->selFlags |= pSub->selFlags & SF_Distinct; + + /* + ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; + ** + ** One is tempted to try to add a and b to combine the limits. But this + ** does not work if either limit is negative. + */ + if( pSub->pLimit ){ + pParent->pLimit = pSub->pLimit; + pSub->pLimit = 0; + } + } -/* -** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to -** a dynamically allocated instance of the following structure. -*/ -struct WhereOrInfo { - WhereClause wc; /* Decomposition into subterms */ - Bitmask indexable; /* Bitmask of all indexable tables in the clause */ -}; + /* Finially, delete what is left of the subquery and return + ** success. + */ + sqlite3SelectDelete(db, pSub1); -/* -** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to -** a dynamically allocated instance of the following structure. -*/ -struct WhereAndInfo { - WhereClause wc; /* The subexpression broken out */ -}; + return 1; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** An instance of the following structure keeps track of a mapping -** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. -** -** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE -** clause, the cursor numbers might not begin with 0 and they might -** contain gaps in the numbering sequence. But we want to make maximum -** use of the bits in our bitmasks. This structure provides a mapping -** from the sparse cursor numbers into consecutive integers beginning -** with 0. -** -** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask -** corresponds VDBE cursor number B. The A-th bit of a bitmask is 1<3, 5->1, 8->2, 29->0, -** 57->5, 73->4. Or one of 719 other combinations might be used. It -** does not really matter. What is important is that sparse cursor -** numbers all get mapped into bit numbers that begin with 0 and contain -** no gaps. -*/ -struct WhereMaskSet { - int n; /* Number of assigned cursor values */ - int ix[BMS]; /* Cursor assigned to each bit */ -}; - -/* -** A WhereCost object records a lookup strategy and the estimated -** cost of pursuing that strategy. -*/ -struct WhereCost { - WherePlan plan; /* The lookup strategy */ - double rCost; /* Overall cost of pursuing this search strategy */ - Bitmask used; /* Bitmask of cursors used by this plan */ -}; - -/* -** Bitmasks for the operators that indices are able to exploit. An -** OR-ed combination of these values can be used when searching for -** terms in the where clause. +** 2. There is a single expression in the result set, and it is +** either min(x) or max(x), where x is a column reference. */ -#define WO_IN 0x001 -#define WO_EQ 0x002 -#define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) -#define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) -#define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) -#define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x040 -#define WO_ISNULL 0x080 -#define WO_OR 0x100 /* Two or more OR-connected terms */ -#define WO_AND 0x200 /* Two or more AND-connected terms */ +static u8 minMaxQuery(Select *p){ + Expr *pExpr; + ExprList *pEList = p->pEList; -#define WO_ALL 0xfff /* Mask of all possible WO_* values */ -#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ + if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; + pExpr = pEList->a[0].pExpr; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; + if( NEVER(ExprHasProperty(pExpr, EP_xIsSelect)) ) return 0; + pEList = pExpr->x.pList; + if( pEList==0 || pEList->nExpr!=1 ) return 0; + if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + if( sqlite3StrICmp(pExpr->u.zToken,"min")==0 ){ + return WHERE_ORDERBY_MIN; + }else if( sqlite3StrICmp(pExpr->u.zToken,"max")==0 ){ + return WHERE_ORDERBY_MAX; + } + return WHERE_ORDERBY_NORMAL; +} /* -** Value for wsFlags returned by bestIndex() and stored in -** WhereLevel.wsFlags. These flags determine which search -** strategies are appropriate. +** The select statement passed as the first argument is an aggregate query. +** The second argment is the associated aggregate-info object. This +** function tests if the SELECT is of the form: ** -** The least significant 12 bits is reserved as a mask for WO_ values above. -** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL. -** But if the table is the right table of a left join, WhereLevel.wsFlags -** is set to WO_IN|WO_EQ. The WhereLevel.wsFlags field can then be used as -** the "op" parameter to findTerm when we are resolving equality constraints. -** ISNULL constraints will then not be used on the right table of a left -** join. Tickets #2177 and #2189. +** SELECT count(*) FROM +** +** where table is a database table, not a sub-select or view. If the query +** does match this pattern, then a pointer to the Table object representing +** is returned. Otherwise, 0 is returned. */ -#define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ -#define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ -#define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ -#define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ -#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ -#define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xpParse = pParse; - pWC->pMaskSet = pMaskSet; - pWC->nTerm = 0; - pWC->nSlot = ArraySize(pWC->aStatic); - pWC->a = pWC->aStatic; - pWC->vmask = 0; -} + assert( !p->pGroupBy ); -/* Forward reference */ -static void whereClauseClear(WhereClause*); + if( p->pWhere || p->pEList->nExpr!=1 + || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect + ){ + return 0; + } + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + assert( pTab && !pTab->pSelect && pExpr ); -/* -** Deallocate all memory associated with a WhereOrInfo object. -*/ -static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ - whereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + if( IsVirtual(pTab) ) return 0; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; + if( (pAggInfo->aFunc[0].pFunc->flags&SQLITE_FUNC_COUNT)==0 ) return 0; + if( pExpr->flags&EP_Distinct ) return 0; -/* -** Deallocate all memory associated with a WhereAndInfo object. -*/ -static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ - whereClauseClear(&p->wc); - sqlite3DbFree(db, p); + return pTab; } /* -** Deallocate a WhereClause structure. The WhereClause structure -** itself is not freed. This routine is the inverse of whereClauseInit(). +** If the source-list item passed as an argument was augmented with an +** INDEXED BY clause, then try to locate the specified index. If there +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** pFrom->pIndex and return SQLITE_OK. */ -static void whereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; - sqlite3 *db = pWC->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); - } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ + if( pFrom->pTab && pFrom->zIndex ){ + Table *pTab = pFrom->pTab; + char *zIndex = pFrom->zIndex; + Index *pIdx; + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndex); + pIdx=pIdx->pNext + ); + if( !pIdx ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); + pParse->checkSchema = 1; + return SQLITE_ERROR; } + pFrom->pIndex = pIdx; } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); - } + return SQLITE_OK; } /* -** Add a single new WhereTerm entry to the WhereClause object pWC. -** The new WhereTerm object is constructed from Expr p and with wtFlags. -** The index in pWC->a[] of the new WhereTerm is returned on success. -** 0 is returned if the new WhereTerm could not be added due to a memory -** allocation error. The memory allocation failure will be recorded in -** the db->mallocFailed flag so that higher-level functions can detect it. +** This routine is a Walker callback for "expanding" a SELECT statement. +** "Expanding" means to do the following: ** -** This routine will increase the size of the pWC->a[] array as necessary. +** (1) Make sure VDBE cursor numbers have been assigned to every +** element of the FROM clause. ** -** If the wtFlags argument includes TERM_DYNAMIC, then responsibility -** for freeing the expression p is assumed by the WhereClause object pWC. -** This is true even if this routine fails to allocate a new WhereTerm. +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** defines FROM clause. When views appear in the FROM clause, +** fill pTabList->a[].pSelect with a copy of the SELECT statement +** that implements the view. A copy is made of the view's SELECT +** statement so that we can freely modify or delete that statement +** without worrying about messing up the presistent representation +** of the view. +** +** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** on joins and the ON and USING clause of joins. +** +** (4) Scan the list of columns in the result set (pEList) looking +** for instances of the "*" operator or the TABLE.* operator. +** If found, expand each "*" to be every column in every table +** and TABLE.* to be every column in TABLE. ** -** WARNING: This routine might reallocate the space used to store -** WhereTerms. All pointers to WhereTerms should be invalidated after -** calling this routine. Such pointers may be reinitialized by referencing -** the pWC->a[] array. */ -static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ - WhereTerm *pTerm; - int idx; - testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ - if( pWC->nTerm>=pWC->nSlot ){ - WhereTerm *pOld = pWC->a; - sqlite3 *db = pWC->pParse->db; - pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); - if( pWC->a==0 ){ - if( wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, p); - } - pWC->a = pOld; - return 0; - } - memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); - if( pOld!=pWC->aStatic ){ - sqlite3DbFree(db, pOld); +static int selectExpander(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i, j, k; + SrcList *pTabList; + ExprList *pEList; + struct SrcList_item *pFrom; + sqlite3 *db = pParse->db; + + if( db->mallocFailed ){ + return WRC_Abort; + } + if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){ + return WRC_Prune; + } + p->selFlags |= SF_Expanded; + pTabList = p->pSrc; + pEList = p->pEList; + + /* Make sure cursor numbers have been assigned to all entries in + ** the FROM clause of the SELECT statement. + */ + sqlite3SrcListAssignCursors(pParse, pTabList); + + /* Look up every table named in the FROM clause of the select. If + ** an entry of the FROM clause is a subquery instead of a table or view, + ** then create a transient table structure to describe the subquery. + */ + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab; + if( pFrom->pTab!=0 ){ + /* This statement has already been prepared. There is no need + ** to go further. */ + assert( i==0 ); + return WRC_Prune; + } + if( pFrom->zName==0 ){ +#ifndef SQLITE_OMIT_SUBQUERY + Select *pSel = pFrom->pSelect; + /* A sub-query in the FROM clause of a SELECT */ + assert( pSel!=0 ); + assert( pFrom->pTab==0 ); + sqlite3WalkSelect(pWalker, pSel); + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); + while( pSel->pPrior ){ pSel = pSel->pPrior; } + selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + pTab->iPKey = -1; + pTab->nRowEst = 1000000; + pTab->tabFlags |= TF_Ephemeral; +#endif + }else{ + /* An ordinary table or view name in the FROM clause */ + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = + sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + if( pTab==0 ) return WRC_Abort; + pTab->nRef++; +#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) + if( pTab->pSelect || IsVirtual(pTab) ){ + /* We reach here if the named table is a really a view */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; + assert( pFrom->pSelect==0 ); + pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + sqlite3WalkSelect(pWalker, pFrom->pSelect); + } +#endif + } + + /* Locate the index named by the INDEXED BY clause, if any. */ + if( sqlite3IndexedByLookup(pParse, pFrom) ){ + return WRC_Abort; } - pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); } - pTerm = &pWC->a[idx = pWC->nTerm++]; - pTerm->pExpr = p; - pTerm->wtFlags = wtFlags; - pTerm->pWC = pWC; - pTerm->iParent = -1; - return idx; -} -/* -** This routine identifies subexpressions in the WHERE clause where -** each subexpression is separated by the AND operator or some other -** operator specified in the op parameter. The WhereClause structure -** is filled with pointers to subexpressions. For example: -** -** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) -** \________/ \_______________/ \________________/ -** slot[0] slot[1] slot[2] -** -** The original WHERE clause in pExpr is unaltered. All this routine -** does is make slot[] entries point to substructure within pExpr. -** -** In the previous sentence and in the diagram, "slot[]" refers to -** the WhereClause.a[] array. The slot[] array grows as needed to contain -** all terms of the WHERE clause. -*/ -static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ - pWC->op = (u8)op; - if( pExpr==0 ) return; - if( pExpr->op!=op ){ - whereClauseInsert(pWC, pExpr, 0); - }else{ - whereSplit(pWC, pExpr->pLeft, op); - whereSplit(pWC, pExpr->pRight, op); + /* Process NATURAL keywords, and ON and USING clauses of joins. + */ + if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + return WRC_Abort; } -} -/* -** Initialize an expression mask set (a WhereMaskSet object) -*/ -#define initMaskSet(P) memset(P, 0, sizeof(*P)) + /* For every "*" that occurs in the column list, insert the names of + ** all columns in all tables. And for every TABLE.* insert the names + ** of all columns in TABLE. The parser inserted a special expression + ** with the TK_ALL operator for each "*" that it found in the column list. + ** The following code just has to locate the TK_ALL expressions and expand + ** each one to the list of all columns in all tables. + ** + ** The first loop just checks to see if there are any "*" operators + ** that need expanding. + */ + for(k=0; knExpr; k++){ + Expr *pE = pEList->a[k].pExpr; + if( pE->op==TK_ALL ) break; + assert( pE->op!=TK_DOT || pE->pRight!=0 ); + assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); + if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break; + } + if( knExpr ){ + /* + ** If we get here it means the result set contains one or more "*" + ** operators that need to be expanded. Loop through each expression + ** in the result set and expand them one by one. + */ + struct ExprList_item *a = pEList->a; + ExprList *pNew = 0; + int flags = pParse->db->flags; + int longNames = (flags & SQLITE_FullColNames)!=0 + && (flags & SQLITE_ShortColNames)==0; -/* -** Return the bitmask for the given cursor number. Return 0 if -** iCursor is not in the set. -*/ -static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ - int i; - assert( pMaskSet->n<=sizeof(Bitmask)*8 ); - for(i=0; in; i++){ - if( pMaskSet->ix[i]==iCursor ){ - return ((Bitmask)1)<nExpr; k++){ + Expr *pE = a[k].pExpr; + assert( pE->op!=TK_DOT || pE->pRight!=0 ); + if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pE->pRight->op!=TK_ALL) ){ + /* This particular expression does not need to be expanded. + */ + pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); + if( pNew ){ + pNew->a[pNew->nExpr-1].zName = a[k].zName; + pNew->a[pNew->nExpr-1].zSpan = a[k].zSpan; + a[k].zName = 0; + a[k].zSpan = 0; + } + a[k].pExpr = 0; + }else{ + /* This expression is a "*" or a "TABLE.*" and needs to be + ** expanded. */ + int tableSeen = 0; /* Set to 1 when TABLE matches */ + char *zTName; /* text of name of TABLE */ + if( pE->op==TK_DOT ){ + assert( pE->pLeft!=0 ); + assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); + zTName = pE->pLeft->u.zToken; + }else{ + zTName = 0; + } + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + char *zTabName = pFrom->zAlias; + if( zTabName==0 ){ + zTabName = pTab->zName; + } + if( db->mallocFailed ) break; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + tableSeen = 1; + for(j=0; jnCol; j++){ + Expr *pExpr, *pRight; + char *zName = pTab->aCol[j].zName; + char *zColname; /* The computed column name */ + char *zToFree; /* Malloced string that needs to be freed */ + Token sColname; /* Computed column name as a token */ + + /* If a column is marked as 'hidden' (currently only possible + ** for virtual tables), do not include it in the expanded + ** result-set list. + */ + if( IsHiddenColumn(&pTab->aCol[j]) ){ + assert(IsVirtual(pTab)); + continue; + } + + if( i>0 && zTName==0 ){ + if( (pFrom->jointype & JT_NATURAL)!=0 + && tableAndColumnIndex(pTabList, i, zName, 0, 0) + ){ + /* In a NATURAL join, omit the join columns from the + ** table to the right of the join */ + continue; + } + if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ + /* In a join with a USING clause, omit columns in the + ** using clause from the table on the right. */ + continue; + } + } + pRight = sqlite3Expr(db, TK_ID, zName); + zColname = zName; + zToFree = 0; + if( longNames || pTabList->nSrc>1 ){ + Expr *pLeft; + pLeft = sqlite3Expr(db, TK_ID, zTabName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( longNames ){ + zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); + zToFree = zColname; + } + }else{ + pExpr = pRight; + } + pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); + sColname.z = zColname; + sColname.n = sqlite3Strlen30(zColname); + sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + sqlite3DbFree(db, zToFree); + } + } + if( !tableSeen ){ + if( zTName ){ + sqlite3ErrorMsg(pParse, "no such table: %s", zTName); + }else{ + sqlite3ErrorMsg(pParse, "no tables specified"); + } + } + } } + sqlite3ExprListDelete(db, pEList); + p->pEList = pNew; } - return 0; +#if SQLITE_MAX_COLUMN + if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + } +#endif + return WRC_Continue; } /* -** Create a new mask for cursor iCursor. +** No-op routine for the parse-tree walker. ** -** There is one cursor per table in the FROM clause. The number of -** tables in the FROM clause is limited by a test early in the -** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] -** array will never overflow. +** When this routine is the Walker.xExprCallback then expression trees +** are walked without any actions being taken at each node. Presumably, +** when this routine is used for Walker.xExprCallback then +** Walker.xSelectCallback is set to do something useful for every +** subquery in the parser tree. */ -static void createMask(WhereMaskSet *pMaskSet, int iCursor){ - assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); - pMaskSet->ix[pMaskSet->n++] = iCursor; +static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; } /* -** This routine walks (recursively) an expression tree and generates -** a bitmask indicating which tables are used in that expression -** tree. +** This routine "expands" a SELECT statement and all of its subqueries. +** For additional information on what it means to "expand" a SELECT +** statement, see the comment on the selectExpand worker callback above. ** -** In order for this routine to work, the calling function must have -** previously invoked sqlite3ResolveExprNames() on the expression. See -** the header comment on that routine for additional information. -** The sqlite3ResolveExprNames() routines looks for column names and -** sets their opcodes to TK_COLUMN and their Expr.iTable fields to -** the VDBE cursor number of the table. This routine just has to -** translate the cursor numbers into bitmask values and OR all -** the bitmasks together. +** Expanding a SELECT statement is the first step in processing a +** SELECT statement. The SELECT statement must be expanded before +** name resolution is performed. +** +** If anything goes wrong, an error message is written into pParse. +** The calling function can detect the problem by looking at pParse->nErr +** and/or pParse->db->mallocFailed. */ -static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); -static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); -static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ - Bitmask mask = 0; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ - mask = getMask(pMaskSet, p->iTable); - return mask; - } - mask = exprTableUsage(pMaskSet, p->pRight); - mask |= exprTableUsage(pMaskSet, p->pLeft); - if( ExprHasProperty(p, EP_xIsSelect) ){ - mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); - }else{ - mask |= exprListTableUsage(pMaskSet, p->x.pList); - } - return mask; +static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ + Walker w; + w.xSelectCallback = selectExpander; + w.xExprCallback = exprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); } -static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() +** interface. +** +** For each FROM-clause subquery, add Column.zType and Column.zColl +** information to the Table structure that represents the result set +** of that subquery. +** +** The Table structure that represents the result set was constructed +** by selectExpander() but the type and collation information was omitted +** at that point because identifiers had not yet been resolved. This +** routine is called after identifier resolution. +*/ +static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ + Parse *pParse; int i; - Bitmask mask = 0; - if( pList ){ - for(i=0; inExpr; i++){ - mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); + SrcList *pTabList; + struct SrcList_item *pFrom; + + assert( p->selFlags & SF_Resolved ); + if( (p->selFlags & SF_HasTypeInfo)==0 ){ + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + assert( pSel ); + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); + } } } - return mask; -} -static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ - Bitmask mask = 0; - while( pS ){ - mask |= exprListTableUsage(pMaskSet, pS->pEList); - mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); - mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); - mask |= exprTableUsage(pMaskSet, pS->pWhere); - mask |= exprTableUsage(pMaskSet, pS->pHaving); - pS = pS->pPrior; - } - return mask; + return WRC_Continue; } +#endif + /* -** Return TRUE if the given operator is one of the operators that is -** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", and "IN". +** This routine adds datatype and collating sequence information to +** the Table structures of all FROM-clause subqueries in a +** SELECT statement. ** -** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be -** of one of the following forms: column = expression column > expression -** column >= expression column < expression column <= expression -** expression = column expression > column expression >= column -** expression < column expression <= column column IN -** (expression-list) column IN (subquery) column IS NULL +** Use this routine after name resolution. */ -static int allowedOp(int op){ - assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; +static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ +#ifndef SQLITE_OMIT_SUBQUERY + Walker w; + w.xSelectCallback = selectAddSubqueryTypeInfo; + w.xExprCallback = exprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); +#endif } -/* -** Swap two objects of type TYPE. -*/ -#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} /* -** Commute a comparison operator. Expressions of the form "X op Y" -** are converted into "Y op X". +** This routine sets of a SELECT statement for processing. The +** following is accomplished: ** -** If a collation sequence is associated with either the left or right -** side of the comparison, it remains associated with the same side after -** the commutation. So "Y collate NOCASE op X" becomes -** "X collate NOCASE op Y". This is because any collation sequence on -** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_ExpCollate flag -** is not commuted. +** * VDBE Cursor numbers are assigned to all FROM-clause terms. +** * Ephemeral Table objects are created for all FROM-clause subqueries. +** * ON and USING clauses are shifted into WHERE statements +** * Wildcards "*" and "TABLE.*" in result sets are expanded. +** * Identifiers in expression are matched to tables. +** +** This routine acts recursively on all subqueries within the SELECT. */ -static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_ExpCollate); - u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate); - assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - pExpr->pRight->pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); - pExpr->pLeft->pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl); - pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft; - pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight; - SWAP(Expr*,pExpr->pRight,pExpr->pLeft); - if( pExpr->op>=TK_GT ){ - assert( TK_LT==TK_GT+2 ); - assert( TK_GE==TK_LE+2 ); - assert( TK_GT>TK_EQ ); - assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); - pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; - } +SQLITE_PRIVATE void sqlite3SelectPrep( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for container */ +){ + sqlite3 *db; + if( NEVER(p==0) ) return; + db = pParse->db; + if( p->selFlags & SF_HasTypeInfo ) return; + sqlite3SelectExpand(pParse, p); + if( pParse->nErr || db->mallocFailed ) return; + sqlite3ResolveSelectNames(pParse, p, pOuterNC); + if( pParse->nErr || db->mallocFailed ) return; + sqlite3SelectAddTypeInfo(pParse, p); } /* -** Translate from TK_xx operator to WO_xx bitmask. +** Reset the aggregate accumulator. +** +** The aggregate accumulator is a set of memory cells that hold +** intermediate results while calculating an aggregate. This +** routine simply stores NULLs in all of those memory cells. */ -static u16 operatorMask(int op){ - u16 c; - assert( allowedOp(op) ); - if( op==TK_IN ){ - c = WO_IN; - }else if( op==TK_ISNULL ){ - c = WO_ISNULL; - }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); +static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pFunc; + if( pAggInfo->nFunc+pAggInfo->nColumn==0 ){ + return; + } + for(i=0; inColumn; i++){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pAggInfo->aCol[i].iMem); + } + for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pFunc->iMem); + if( pFunc->iDistinct>=0 ){ + Expr *pE = pFunc->pExpr; + assert( !ExprHasProperty(pE, EP_xIsSelect) ); + if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " + "argument"); + pFunc->iDistinct = -1; + }else{ + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + } + } } - assert( op!=TK_ISNULL || c==WO_ISNULL ); - assert( op!=TK_IN || c==WO_IN ); - assert( op!=TK_EQ || c==WO_EQ ); - assert( op!=TK_LT || c==WO_LT ); - assert( op!=TK_LE || c==WO_LE ); - assert( op!=TK_GT || c==WO_GT ); - assert( op!=TK_GE || c==WO_GE ); - return c; } /* -** Search for a term in the WHERE clause that is of the form "X " -** where X is a reference to the iColumn of table iCur and is one of -** the WO_xx operator codes specified by the op parameter. -** Return a pointer to the term. Return 0 if not found. +** Invoke the OP_AggFinalize opcode for every aggregate function +** in the AggInfo structure. */ -static WhereTerm *findTerm( - WhereClause *pWC, /* The WHERE clause to be searched */ - int iCur, /* Cursor number of LHS */ - int iColumn, /* Column number of LHS */ - Bitmask notReady, /* RHS must not overlap with this mask */ - u32 op, /* Mask of WO_xx values describing operator */ - Index *pIdx /* Must be compatible with this index, if not NULL */ -){ - WhereTerm *pTerm; - int k; - assert( iCur>=0 ); - op &= WO_ALL; - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; - - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - assert(pColl || pParse->nErr); - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; - } - if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; - } - return pTerm; - } - } - return 0; -} - -/* Forward reference */ -static void exprAnalyze(SrcList*, WhereClause*, int); - -/* -** Call exprAnalyze on all terms in a WHERE clause. -** -** -*/ -static void exprAnalyzeAll( - SrcList *pTabList, /* the FROM clause */ - WhereClause *pWC /* the WHERE clause to be analyzed */ -){ +static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; int i; - for(i=pWC->nTerm-1; i>=0; i--){ - exprAnalyze(pTabList, pWC, i); + struct AggInfo_func *pF; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + ExprList *pList = pF->pExpr->x.pList; + assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); + sqlite3VdbeAddOp4(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, 0, + (void*)pF->pFunc, P4_FUNCDEF); } } -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* -** Check to see if the given expression is a LIKE or GLOB operator that -** can be optimized using inequality constraints. Return TRUE if it is -** so and false if not. -** -** In order for the operator to be optimizible, the RHS must be a string -** literal that does not begin with a wildcard. +** Update the accumulator memory cells for an aggregate based on +** the current cursor position. */ -static int isLikeOrGlob( - Parse *pParse, /* Parsing and code generating context */ - Expr *pExpr, /* Test this expression */ - Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ - int *pisComplete, /* True if the only wildcard is % in the last character */ - int *pnoCase /* True if uppercase is equivalent to lowercase */ -){ - const char *z = 0; /* String on RHS of LIKE operator */ - Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ - ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ - int cnt; /* Number of non-wildcard prefix characters */ - char wc[3]; /* Wildcard characters */ - sqlite3 *db = pParse->db; /* Database connection */ - sqlite3_value *pVal = 0; - int op; /* Opcode of pRight */ - - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ - return 0; - } -#ifdef SQLITE_EBCDIC - if( *pnoCase ) return 0; -#endif - pList = pExpr->x.pList; - pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; - } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ +static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pF; + struct AggInfo_col *pC; - pRight = pList->a[0].pExpr; - op = pRight->op; - if( op==TK_REGISTER ){ - op = pRight->op2; - } - if( op==TK_VARIABLE ){ - Vdbe *pReprepare = pParse->pReprepare; - int iCol = pRight->iColumn; - pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE); - if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); + pAggInfo->directMode = 1; + sqlite3ExprCacheClear(pParse); + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + int nArg; + int addrNext = 0; + int regAgg; + ExprList *pList = pF->pExpr->x.pList; + assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); + if( pList ){ + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 1); + }else{ + nArg = 0; + regAgg = 0; } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ - assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); - }else if( op==TK_STRING ){ - z = pRight->u.zToken; - } - if( z ){ - cnt = 0; - while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ - cnt++; + if( pF->iDistinct>=0 ){ + addrNext = sqlite3VdbeMakeLabel(v); + assert( nArg==1 ); + codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); } - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ - Expr *pPrefix; - *pisComplete = c==wc[0] && z[cnt+1]==0; - pPrefix = sqlite3Expr(db, TK_STRING, z); - if( pPrefix ) pPrefix->u.zToken[cnt] = 0; - *ppPrefix = pPrefix; - if( op==TK_VARIABLE ){ - Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ - if( *pisComplete && pRight->u.zToken[1] ){ - /* If the rhs of the LIKE expression is a variable, and the current - ** value of the variable means there is no need to invoke the LIKE - ** function, then no OP_Variable will be added to the program. - ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To workaround them, add a dummy OP_Variable here. - */ - int r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCodeTarget(pParse, pRight, r1); - sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); - sqlite3ReleaseTempReg(pParse, r1); - } + if( pF->pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); } - }else{ - z = 0; + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, + (void*)pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + if( addrNext ){ + sqlite3VdbeResolveLabel(v, addrNext); + sqlite3ExprCacheClear(pParse); } } - sqlite3ValueFree(pVal); - return (z!=0); + /* Before populating the accumulator registers, clear the column cache. + ** Otherwise, if any of the required column values are already present + ** in registers, sqlite3ExprCode() may use OP_SCopy to copy the value + ** to pC->iMem. But by the time the value is used, the original register + ** may have been used, invalidating the underlying buffer holding the + ** text or blob value. See ticket [883034dcb5]. + ** + ** Another solution would be to change the OP_SCopy used to copy cached + ** values to an OP_Copy. + */ + sqlite3ExprCacheClear(pParse); + for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ + sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); + } + pAggInfo->directMode = 0; + sqlite3ExprCacheClear(pParse); } -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ - -#ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Check to see if the given expression is of the form -** -** column MATCH expr -** -** If it is then return TRUE. If not, return FALSE. +** Add a single OP_Explain instruction to the VDBE to explain a simple +** count(*) query ("SELECT count(*) FROM pTab"). */ -static int isMatchOfColumn( - Expr *pExpr /* Test this expression */ +#ifndef SQLITE_OMIT_EXPLAIN +static void explainSimpleCount( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being queried */ + Index *pIdx /* Index used to optimize scan, or NULL */ ){ - ExprList *pList; - - if( pExpr->op!=TK_FUNCTION ){ - return 0; - } - if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ - return 0; - } - pList = pExpr->x.pList; - if( pList->nExpr!=2 ){ - return 0; - } - if( pList->a[1].pExpr->op != TK_COLUMN ){ - return 0; + if( pParse->explain==2 ){ + char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s %s%s(~%d rows)", + pTab->zName, + pIdx ? "USING COVERING INDEX " : "", + pIdx ? pIdx->zName : "", + pTab->nRowEst + ); + sqlite3VdbeAddOp4( + pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC + ); } - return 1; -} -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/* -** If the pBase expression originated in the ON or USING clause of -** a join, then transfer the appropriate markings over to derived. -*/ -static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; } +#else +# define explainSimpleCount(a,b,c) +#endif -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* -** Analyze a term that consists of two or more OR-connected -** subterms. So in: -** -** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) -** ^^^^^^^^^^^^^^^^^^^^ -** -** This routine analyzes terms such as the middle term in the above example. -** A WhereOrTerm object is computed and attached to the term under -** analysis, regardless of the outcome of the analysis. Hence: -** -** WhereTerm.wtFlags |= TERM_ORINFO -** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object -** -** The term being analyzed must have two or more of OR-connected subterms. -** A single subterm might be a set of AND-connected sub-subterms. -** Examples of terms under analysis: -** -** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 -** (B) x=expr1 OR expr2=x OR x=expr3 -** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) -** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') -** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) -** -** CASE 1: -** -** If all subterms are of the form T.C=expr for some single column of C -** a single table T (as shown in example B above) then create a new virtual -** term that is an equivalent IN expression. In other words, if the term -** being analyzed is: +** Generate code for the SELECT statement given in the p argument. ** -** x = expr1 OR expr2 = x OR x = expr3 +** The results are distributed in various ways depending on the +** contents of the SelectDest structure pointed to by argument pDest +** as follows: ** -** then create a new virtual term like this: +** pDest->eDest Result +** ------------ ------------------------------------------- +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. ** -** x IN (expr1,expr2,expr3) +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iParm then abandon the rest +** of the query. This destination implies "LIMIT 1". ** -** CASE 2: +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iParm. +** Apply the affinity pDest->affinity before storing +** results. Used to implement "IN (SELECT ...)". ** -** If all subterms are indexable by a single table T, then set +** SRT_Union Store results as a key in a temporary table pDest->iParm. ** -** WhereTerm.eOperator = WO_OR -** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** SRT_Except Remove results from the temporary table pDest->iParm. ** -** A subterm is "indexable" if it is of the form -** "T.C " where C is any column of table T and -** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". -** A subterm is also indexable if it is an AND of two or more -** subsubterms at least one of which is indexable. Indexable AND -** subterms have their eOperator set to WO_AND and they have -** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** SRT_Table Store results in temporary table pDest->iParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. ** -** From another point of view, "indexable" means that the subterm could -** potentially be used with an index if an appropriate index exists. -** This analysis does not consider whether or not the index exists; that -** is something the bestIndex() routine will determine. This analysis -** only looks at whether subterms appropriate for indexing exist. +** SRT_EphemTab Create an temporary table pDest->iParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. ** -** All examples A through E above all satisfy case 2. But if a term -** also statisfies case 1 (such as B) we know that the optimizer will -** always prefer case 1, so in that case we pretend that case 2 is not -** satisfied. +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iParm. ** -** It might be the case that multiple tables are indexable. For example, -** (E) above is indexable on tables P, Q, and R. +** SRT_Exists Store a 1 in memory cell pDest->iParm if the result +** set is not empty. ** -** Terms that satisfy case 2 are candidates for lookup by using -** separate indices to find rowids for each subterm and composing -** the union of all rowids using a RowSet object. This is similar -** to "bitmap indices" in other database engines. +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. ** -** OTHERWISE: +** This routine returns the number of errors. If any errors are +** encountered, then an appropriate error message is left in +** pParse->zErrMsg. ** -** If neither case 1 nor case 2 apply, then leave the eOperator set to -** zero. This term is not useful for search. +** This routine does NOT free the Select structure passed in. The +** calling function needs to do that. */ -static void exprAnalyzeOrTerm( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the complete WHERE clause */ - int idxTerm /* Index of the OR-term to be analyzed */ +SQLITE_PRIVATE int sqlite3Select( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + SelectDest *pDest /* What to do with the query results */ ){ - Parse *pParse = pWC->pParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ - Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */ - int i; /* Loop counters */ - WhereClause *pOrWc; /* Breakup of pTerm into subterms */ - WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ - WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ - Bitmask chngToIN; /* Tables that might satisfy case 1 */ - Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + int i, j; /* Loop counters */ + WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ + Vdbe *v; /* The virtual machine under construction */ + int isAgg; /* True for select lists like "count(*)" */ + ExprList *pEList; /* List of columns to extract. */ + SrcList *pTabList; /* List of tables to select from */ + Expr *pWhere; /* The WHERE clause. May be NULL */ + ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ + ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ + Expr *pHaving; /* The HAVING clause. May be NULL */ + int isDistinct; /* True if the DISTINCT keyword is present */ + int distinct; /* Table to use for the distinct set */ + int rc = 1; /* Value to return from this function */ + int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ + AggInfo sAggInfo; /* Information used by aggregate queries */ + int iEnd; /* Address of the end of the query */ + sqlite3 *db; /* The database connection */ - /* - ** Break the OR clause into its separate subterms. The subterms are - ** stored in a WhereClause structure containing within the WhereOrInfo - ** object that is attached to the original OR clause term. +#ifndef SQLITE_OMIT_EXPLAIN + int iRestoreSelectId = pParse->iSelectId; + pParse->iSelectId = pParse->iNextSelectId++; +#endif + + db = pParse->db; + if( p==0 || db->mallocFailed || pParse->nErr ){ + return 1; + } + if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; + memset(&sAggInfo, 0, sizeof(sAggInfo)); + + if( IgnorableOrderby(pDest) ){ + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); + /* If ORDER BY makes no difference in the output then neither does + ** DISTINCT so it can be removed too. */ + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + p->selFlags &= ~SF_Distinct; + } + sqlite3SelectPrep(pParse, p, 0); + pOrderBy = p->pOrderBy; + pTabList = p->pSrc; + pEList = p->pEList; + if( pParse->nErr || db->mallocFailed ){ + goto select_end; + } + isAgg = (p->selFlags & SF_Aggregate)!=0; + assert( pEList!=0 ); + + /* Begin generating code. */ - assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); - assert( pExpr->op==TK_OR ); - pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); - if( pOrInfo==0 ) return; - pTerm->wtFlags |= TERM_ORINFO; - pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWC->pParse, pMaskSet); - whereSplit(pOrWc, pExpr, TK_OR); - exprAnalyzeAll(pSrc, pOrWc); - if( db->mallocFailed ) return; - assert( pOrWc->nTerm>=2 ); + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto select_end; - /* - ** Compute the set of tables that might satisfy cases 1 or 2. + /* If writing to memory or generating a set + ** only a single column may be output. */ - indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ - if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ - WhereAndInfo *pAndInfo; - assert( pOrTerm->eOperator==0 ); - assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); - chngToIN = 0; - pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); - if( pAndInfo ){ - WhereClause *pAndWC; - WhereTerm *pAndTerm; - int j; - Bitmask b = 0; - pOrTerm->u.pAndInfo = pAndInfo; - pOrTerm->wtFlags |= TERM_ANDINFO; - pOrTerm->eOperator = WO_AND; - pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pParse, pMaskSet); - whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); - exprAnalyzeAll(pSrc, pAndWC); - testcase( db->mallocFailed ); - if( !db->mallocFailed ){ - for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ - assert( pAndTerm->pExpr ); - if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(pMaskSet, pAndTerm->leftCursor); - } - } - } - indexable &= b; - } - }else if( pOrTerm->wtFlags & TERM_COPIED ){ - /* Skip this term for now. We revisit it when we process the - ** corresponding TERM_VIRTUAL term */ - }else{ - Bitmask b; - b = getMask(pMaskSet, pOrTerm->leftCursor); - if( pOrTerm->wtFlags & TERM_VIRTUAL ){ - WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(pMaskSet, pOther->leftCursor); - } - indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ - chngToIN = 0; - }else{ - chngToIN &= b; - } - } +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; } +#endif - /* - ** Record the set of tables that satisfy case 2. The set might be - ** empty. + /* Generate code for all sub-queries in the FROM clause */ - pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; !p->pPrior && inSrc; i++){ + struct SrcList_item *pItem = &pTabList->a[i]; + SelectDest dest; + Select *pSub = pItem->pSelect; + int isAggSub; - /* - ** chngToIN holds a set of tables that *might* satisfy case 1. But - ** we have to do some additional checking to see if case 1 really - ** is satisfied. - ** - ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means - ** that there is no possibility of transforming the OR clause into an - ** IN operator because one or more terms in the OR clause contain - ** something other than == on a column in the single table. The 1-bit - ** case means that every term of the OR clause is of the form - ** "table.column=expr" for some single table. The one bit that is set - ** will correspond to the common table. We still need to check to make - ** sure the same column is used on all terms. The 2-bit case is when - ** the all terms are of the form "table1.column=table2.column". It - ** might be possible to form an IN operator with either table1.column - ** or table2.column as the LHS if either is common to every term of - ** the OR clause. - ** - ** Note that terms of the form "table.column1=table.column2" (the - ** same table on both sizes of the ==) cannot be optimized. - */ - if( chngToIN ){ - int okToChngToIN = 0; /* True if the conversion to IN is valid */ - int iColumn = -1; /* Column index on lhs of IN operator */ - int iCursor = -1; /* Table cursor common to all terms */ - int j = 0; /* Loop counter */ - - /* Search for a table and column that appears on one side or the - ** other of the == operator in every subterm. That table and column - ** will be recorded in iCursor and iColumn. There might not be any - ** such table and column. Set okToChngToIN if an appropriate table - ** and column is found but leave okToChngToIN false if not found. - */ - for(j=0; j<2 && !okToChngToIN; j++){ - pOrTerm = pOrWc->a; - for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); - pOrTerm->wtFlags &= ~TERM_OR_OK; - if( pOrTerm->leftCursor==iCursor ){ - /* This is the 2-bit case and we are on the second iteration and - ** current term is from the first iteration. So skip this term. */ - assert( j==1 ); - continue; - } - if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ){ - /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceeded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term - ** and use its inversion. */ - testcase( pOrTerm->wtFlags & TERM_COPIED ); - testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); - assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); - continue; - } - iColumn = pOrTerm->u.leftColumn; - iCursor = pOrTerm->leftCursor; - break; - } - if( i<0 ){ - /* No candidate table+column was found. This can only occur - ** on the second iteration */ - assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); - assert( chngToIN==getMask(pMaskSet, iCursor) ); - break; - } - testcase( j==1 ); - - /* We have found a candidate table and column. Check to see if that - ** table and column is common to every term in the OR clause */ - okToChngToIN = 1; - for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); - if( pOrTerm->leftCursor!=iCursor ){ - pOrTerm->wtFlags &= ~TERM_OR_OK; - }else if( pOrTerm->u.leftColumn!=iColumn ){ - okToChngToIN = 0; - }else{ - int affLeft, affRight; - /* If the right-hand side is also a column, then the affinities - ** of both right and left sides must be such that no type - ** conversions are required on the right. (Ticket #2249) - */ - affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); - affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); - if( affRight!=0 && affRight!=affLeft ){ - okToChngToIN = 0; - }else{ - pOrTerm->wtFlags |= TERM_OR_OK; - } - } - } - } + if( pSub==0 || pItem->isPopulated ) continue; - /* At this point, okToChngToIN is true if original pTerm satisfies - ** case 1. In that case, construct a new virtual term that is - ** pTerm converted into an IN operator. - ** - ** EV: R-00211-15100 + /* Increment Parse.nHeight by the height of the largest expression + ** tree refered to by this, the parent select. The child select + ** may contain expression trees of at most + ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit + ** more conservative than necessary, but much easier than enforcing + ** an exact limit. */ - if( okToChngToIN ){ - Expr *pDup; /* A transient duplicate expression */ - ExprList *pList = 0; /* The RHS of the IN operator */ - Expr *pLeft = 0; /* The LHS of the IN operator */ - Expr *pNew; /* The complete IN operator */ + pParse->nHeight += sqlite3SelectExprHeight(p); - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ - if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); - assert( pOrTerm->leftCursor==iCursor ); - assert( pOrTerm->u.leftColumn==iColumn ); - pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup); - pLeft = pOrTerm->pExpr->pLeft; - } - assert( pLeft!=0 ); - pDup = sqlite3ExprDup(db, pLeft, 0); - pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); - if( pNew ){ - int idxNew; - transferJoinMarkings(pNew, pExpr); - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - pNew->x.pList = pList; - idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; - }else{ - sqlite3ExprListDelete(db, pList); + /* Check to see if the subquery can be absorbed into the parent. */ + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; } - pTerm->eOperator = 0; /* case 1 trumps case 2 */ - } - } -} -#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ - - -/* -** The input to this routine is an WhereTerm structure with only the -** "pExpr" field filled in. The job of this routine is to analyze the -** subexpression and populate all the other fields of the WhereTerm -** structure. -** -** If the expression is of the form " X" it gets commuted -** to the standard form of "X ". -** -** If the expression is of the form "X Y" where both X and Y are -** columns, then the original expression is unchanged and a new virtual -** term of the form "Y X" is added to the WHERE clause and -** analyzed separately. The original term is marked with TERM_COPIED -** and the new term is marked with TERM_DYNAMIC (because it's pExpr -** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it -** is a commuted copy of a prior term.) The original term has nChild=1 -** and the copy has idxParent set to the index of the original term. -*/ -static void exprAnalyze( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the WHERE clause */ - int idxTerm /* Index of the term to be analyzed */ -){ - WhereTerm *pTerm; /* The term to be analyzed */ - WhereMaskSet *pMaskSet; /* Set of table index masks */ - Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ - Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ - Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ - int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ - int noCase = 0; /* LIKE/GLOB distinguishes case */ - int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWC->pParse; /* Parsing context */ - sqlite3 *db = pParse->db; /* Database connection */ - - if( db->mallocFailed ){ - return; - } - pTerm = &pWC->a[idxTerm]; - pMaskSet = pWC->pMaskSet; - pExpr = pTerm->pExpr; - prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); - op = pExpr->op; - if( op==TK_IN ){ - assert( pExpr->pRight==0 ); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); + i = -1; }else{ - pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + assert( pItem->isPopulated==0 ); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->isPopulated = 1; + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; } - }else if( op==TK_ISNULL ){ - pTerm->prereqRight = 0; - }else{ - pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); - } - prereqAll = exprTableUsage(pMaskSet, pExpr); - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); - prereqAll |= x; - extraRight = x-1; /* ON clause terms may not be used with an index - ** on left table of a LEFT JOIN. Ticket #3015 */ - } - pTerm->prereqAll = prereqAll; - pTerm->leftCursor = -1; - pTerm->iParent = -1; - pTerm->eOperator = 0; - if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pRight; - if( pLeft->op==TK_COLUMN ){ - pTerm->leftCursor = pLeft->iTable; - pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op); + if( /*pParse->nErr ||*/ db->mallocFailed ){ + goto select_end; } - if( pRight && pRight->op==TK_COLUMN ){ - WhereTerm *pNew; - Expr *pDup; - if( pTerm->leftCursor>=0 ){ - int idxNew; - pDup = sqlite3ExprDup(db, pExpr, 0); - if( db->mallocFailed ){ - sqlite3ExprDelete(db, pDup); - return; - } - idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); - if( idxNew==0 ) return; - pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; - pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; - pTerm->wtFlags |= TERM_COPIED; - }else{ - pDup = pExpr; - pNew = pTerm; - } - exprCommute(pParse, pDup); - pLeft = pDup->pLeft; - pNew->leftCursor = pLeft->iTable; - pNew->u.leftColumn = pLeft->iColumn; - testcase( (prereqLeft | extraRight) != prereqLeft ); - pNew->prereqRight = prereqLeft | extraRight; - pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pParse->nHeight -= sqlite3SelectExprHeight(p); + pTabList = p->pSrc; + if( !IgnorableOrderby(pDest) ){ + pOrderBy = p->pOrderBy; } } + pEList = p->pEList; +#endif + pWhere = p->pWhere; + pGroupBy = p->pGroupBy; + pHaving = p->pHaving; + isDistinct = (p->selFlags & SF_Distinct)!=0; -#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION - /* If a term is the BETWEEN operator, create two new virtual terms - ** that define the range that the BETWEEN implements. For example: - ** - ** a BETWEEN b AND c - ** - ** is converted into: - ** - ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) - ** - ** The two new terms are added onto the end of the WhereClause object. - ** The new terms are "dynamic" and are children of the original BETWEEN - ** term. That means that if the BETWEEN term is coded, the children are - ** skipped. Or, if the children are satisfied by an index, the original - ** BETWEEN term is skipped. +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* If there is are a sequence of queries, do the earlier ones first. */ - else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ - ExprList *pList = pExpr->x.pList; - int i; - static const u8 ops[] = {TK_GE, TK_LE}; - assert( pList!=0 ); - assert( pList->nExpr==2 ); - for(i=0; i<2; i++){ - Expr *pNewExpr; - int idxNew; - pNewExpr = sqlite3PExpr(pParse, ops[i], - sqlite3ExprDup(db, pExpr->pLeft, 0), - sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + if( p->pPrior ){ + if( p->pRightmost==0 ){ + Select *pLoop, *pRight = 0; + int cnt = 0; + int mxSelect; + for(pLoop=p; pLoop; pLoop=pLoop->pPrior, cnt++){ + pLoop->pRightmost = p; + pLoop->pNext = pRight; + pRight = pLoop; + } + mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; + if( mxSelect && cnt>mxSelect ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + goto select_end; + } } - pTerm->nChild = 2; + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); + return rc; } -#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ +#endif -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) - /* Analyze a term that is composed of two or more subterms connected by - ** an OR operator. + /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. + ** GROUP BY might use an index, DISTINCT never does. */ - else if( pExpr->op==TK_OR ){ - assert( pWC->op==TK_AND ); - exprAnalyzeOrTerm(pSrc, pWC, idxTerm); - pTerm = &pWC->a[idxTerm]; + assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ + p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); + pGroupBy = p->pGroupBy; + p->selFlags &= ~SF_Distinct; } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION - /* Add constraints to reduce the search space on a LIKE or GLOB - ** operator. - ** - ** A like pattern of the form "x LIKE 'abc%'" is changed into constraints - ** - ** x>='abc' AND x<'abd' AND x LIKE 'abc%' - ** - ** The last character of the prefix "abc" is incremented to form the - ** termination condition "abd". + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then disable the ORDER BY clause since the GROUP BY + ** will cause elements to come out in the correct order. This is + ** an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER + ** to disable this optimization for testing purposes. */ - if( pWC->op==TK_AND - && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) - ){ - Expr *pLeft; /* LHS of LIKE/GLOB operator */ - Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ - Expr *pNewExpr1; - Expr *pNewExpr2; - int idxNew1; - int idxNew2; - CollSeq *pColl; /* Collating sequence to use */ - - pLeft = pExpr->x.pList->a[1].pExpr; - pStr2 = sqlite3ExprDup(db, pStr1, 0); - if( !db->mallocFailed ){ - u8 c, *pC; /* Last character before the first wildcard */ - pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; - c = *pC; - if( noCase ){ - /* The point is to increment the last character before the first - ** wildcard. But if we increment '@', that will push it into the - ** alphabetic range where case conversions will mess up the - ** inequality. To avoid this, make sure to also run the full - ** LIKE on all candidate expressions by clearing the isComplete flag - */ - if( c=='A'-1 ) isComplete = 0; /* EV: R-64339-08207 */ - - - c = sqlite3UpperToLower[c]; - } - *pC = c + 1; - } - pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr1, 0); - idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), - pStr2, 0); - idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew2==0 ); - exprAnalyze(pSrc, pWC, idxNew2); - pTerm = &pWC->a[idxTerm]; - if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; - } + if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0 + && (db->flags & SQLITE_GroupByOrder)==0 ){ + pOrderBy = 0; } -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Add a WO_MATCH auxiliary term to the constraint set if the - ** current expression is of the form: column MATCH expr. - ** This information is used by the xBestIndex methods of - ** virtual tables. The native query optimizer does not attempt - ** to do anything with MATCH functions. + /* If there is an ORDER BY clause, then this sorting + ** index might end up being unused if the data can be + ** extracted in pre-sorted order. If that is the case, then the + ** OP_OpenEphemeral instruction will be changed to an OP_Noop once + ** we figure out that the sorting index is not needed. The addrSortIndex + ** variable is used to facilitate that change. */ - if( isMatchOfColumn(pExpr) ){ - int idxNew; - Expr *pRight, *pLeft; - WhereTerm *pNewTerm; - Bitmask prereqColumn, prereqExpr; + if( pOrderBy ){ + KeyInfo *pKeyInfo; + pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); + pOrderBy->iECursor = pParse->nTab++; + p->addrOpenEphm[2] = addrSortIndex = + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pOrderBy->iECursor, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + }else{ + addrSortIndex = -1; + } - pRight = pExpr->x.pList->a[0].pExpr; - pLeft = pExpr->x.pList->a[1].pExpr; - prereqExpr = exprTableUsage(pMaskSet, pRight); - prereqColumn = exprTableUsage(pMaskSet, pLeft); - if( (prereqExpr & prereqColumn)==0 ){ - Expr *pNewExpr; - pNewExpr = sqlite3PExpr(pParse, TK_MATCH, - 0, sqlite3ExprDup(db, pRight, 0), 0); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; - pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; - } + /* If the output is destined for a temporary table, open that table. + */ + if( pDest->eDest==SRT_EphemTab ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iParm, pEList->nExpr); } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - /* Prevent ON clause terms of a LEFT JOIN from being used to drive - ** an index for tables to the left of the join. + /* Set the limiter. */ - pTerm->prereqRight |= extraRight; -} + iEnd = sqlite3VdbeMakeLabel(v); + p->nSelectRow = (double)LARGEST_INT64; + computeLimitRegisters(pParse, p, iEnd); -/* -** Return TRUE if any of the expressions in pList->a[iFirst...] contain -** a reference to any table other than the iBase table. -*/ -static int referencesOtherTables( - ExprList *pList, /* Search expressions in ths list */ - WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ - int iFirst, /* Be searching with the iFirst-th expression */ - int iBase /* Ignore references to this table */ -){ - Bitmask allowed = ~getMask(pMaskSet, iBase); - while( iFirstnExpr ){ - if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ - return 1; - } + /* Open a virtual index to use for the distinct set. + */ + if( p->selFlags & SF_Distinct ){ + KeyInfo *pKeyInfo; + assert( isAgg || pGroupBy ); + distinct = pParse->nTab++; + pKeyInfo = keyInfoFromExprList(pParse, p->pEList); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + }else{ + distinct = -1; } - return 0; -} + /* Aggregate and non-aggregate queries are handled differently */ + if( !isAgg && pGroupBy==0 ){ + /* This case is for non-aggregate queries + ** Begin the database scan + */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); + if( pWInfo==0 ) goto select_end; + if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; -/* -** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause. If it can, it returns 1. If pIdx cannot satisfy the -** ORDER BY clause, this routine returns 0. -** -** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the -** left-most table in the FROM clause of that same SELECT statement and -** the table has a cursor number of "base". pIdx is an index on pTab. -** -** nEqCol is the number of columns of pIdx that are used as equality -** constraints. Any of these columns may be missing from the ORDER BY -** clause and the match can still be a success. -** -** All terms of the ORDER BY that match against the index must be either -** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE -** index do not need to satisfy this constraint.) The *pbRev value is -** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if -** the ORDER BY clause is all ASC. -*/ -static int isSortingIndex( - Parse *pParse, /* Parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - ExprList *pOrderBy, /* The ORDER BY clause */ - int nEqCol, /* Number of index columns with == constraints */ - int *pbRev /* Set to 1 if ORDER BY is DESC */ -){ - int i, j; /* Loop counters */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ - int nTerm; /* Number of ORDER BY terms */ - struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ - sqlite3 *db = pParse->db; + /* If sorting index that was created by a prior OP_OpenEphemeral + ** instruction ended up not being needed, then change the OP_OpenEphemeral + ** into an OP_Noop. + */ + if( addrSortIndex>=0 && pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, addrSortIndex, 1); + p->addrOpenEphm[2] = -1; + } - assert( pOrderBy!=0 ); - nTerm = pOrderBy->nExpr; - assert( nTerm>0 ); + /* Use the standard inner loop + */ + assert(!isDistinct); + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, + pWInfo->iContinue, pWInfo->iBreak); - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); + }else{ + /* This is the processing for aggregate queries */ + NameContext sNC; /* Name context for processing aggregate information */ + int iAMem; /* First Mem address for storing current GROUP BY */ + int iBMem; /* First Mem address for previous GROUP BY */ + int iUseFlag; /* Mem address holding flag indicating that at least + ** one row of the input to the aggregator has been + ** processed */ + int iAbortFlag; /* Mem address which causes query abort if positive */ + int groupBySort; /* Rows come from source in GROUP BY order */ + int addrEnd; /* End of processing for this SELECT */ - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - for(i=j=0, pTerm=pOrderBy->a; jnColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - const char *zColl; /* Name of the collating sequence for i-th index term */ + /* Remove any and all aliases between the result set and the + ** GROUP BY clause. + */ + if( pGroupBy ){ + int k; /* Loop counter */ + struct ExprList_item *pItem; /* For looping over expression in a list */ - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ - break; - } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - if( pIdx->zName && inColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = pColl->zName; - } - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( inColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; - }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return 0; + for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ + pItem->iAlias = 0; } - } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqCol ){ - if( termSortOrder!=sortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ - return 0; + for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ + pItem->iAlias = 0; } + if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100; }else{ - sortOrder = termSortOrder; + p->nSelectRow = (double)1; } - j++; - pTerm++; - if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* If the indexed column is the primary key and everything matches - ** so far and none of the ORDER BY terms to the right reference other - ** tables in the join, then we are assured that the index can be used - ** to sort because the primary key is unique and so none of the other - ** columns will make any difference - */ - j = nTerm; + + + /* Create a label to jump to when we want to abort the query */ + addrEnd = sqlite3VdbeMakeLabel(v); + + /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in + ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the + ** SELECT statement. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + sNC.pAggInfo = &sAggInfo; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.pGroupBy = pGroupBy; + sqlite3ExprAnalyzeAggList(&sNC, pEList); + sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); + if( pHaving ){ + sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } - } + sAggInfo.nAccumulator = sAggInfo.nColumn; + for(i=0; ix.pList); + } + if( db->mallocFailed ) goto select_end; - *pbRev = sortOrder!=0; - if( j>=nTerm ){ - /* All terms of the ORDER BY clause are covered by this index so - ** this index can be used for sorting. */ - return 1; - } - if( pIdx->onError!=OE_None && i==pIdx->nColumn - && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* All terms of this index match some prefix of the ORDER BY clause - ** and the index is UNIQUE and no terms on the tail of the ORDER BY - ** clause reference other tables in a join. If this is all true then - ** the order by clause is superfluous. */ - return 1; - } - return 0; -} + /* Processing for aggregates with GROUP BY is very different and + ** much more complex than aggregates without a GROUP BY. + */ + if( pGroupBy ){ + KeyInfo *pKeyInfo; /* Keying information for the group by clause */ + int j1; /* A-vs-B comparision jump */ + int addrOutputRow; /* Start of subroutine that outputs a result row */ + int regOutputRow; /* Return address register for output subroutine */ + int addrSetAbort; /* Set the abort flag and return */ + int addrTopOfLoop; /* Top of the input loop */ + int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ + int addrReset; /* Subroutine for resetting the accumulator */ + int regReset; /* Return address register for reset subroutine */ -/* -** Prepare a crude estimate of the logarithm of the input value. -** The results need not be exact. This is only used for estimating -** the total cost of performing operations with O(logN) or O(NlogN) -** complexity. Because N is just a guess, it is no great tragedy if -** logN is a little off. -*/ -static double estLog(double N){ - double logN = 1; - double x = 10; - while( N>x ){ - logN += 1; - x *= 10; - } - return logN; -} + /* If there is a GROUP BY clause we might need a sorting index to + ** implement it. Allocate that sorting index now. If it turns out + ** that we do not need it after all, the OpenEphemeral instruction + ** will be converted into a Noop. + */ + sAggInfo.sortingIdx = pParse->nTab++; + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sAggInfo.sortingIdx, sAggInfo.nSortingColumn, + 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); -/* -** Two routines for printing the content of an sqlite3_index_info -** structure. Used for testing and debugging only. If neither -** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines -** are no-ops. -*/ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_DEBUG) -static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", - i, - p->aConstraint[i].iColumn, - p->aConstraint[i].iTermOffset, - p->aConstraint[i].op, - p->aConstraint[i].usable); - } - for(i=0; inOrderBy; i++){ - sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); - } -} -static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", - i, - p->aConstraintUsage[i].argvIndex, - p->aConstraintUsage[i].omit); - } - sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); - sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); - sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); - sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); -} -#else -#define TRACE_IDX_INPUTS(A) -#define TRACE_IDX_OUTPUTS(A) -#endif + /* Initialize memory locations used by GROUP BY aggregate processing + */ + iUseFlag = ++pParse->nMem; + iAbortFlag = ++pParse->nMem; + regOutputRow = ++pParse->nMem; + addrOutputRow = sqlite3VdbeMakeLabel(v); + regReset = ++pParse->nMem; + addrReset = sqlite3VdbeMakeLabel(v); + iAMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + iBMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); + VdbeComment((v, "clear abort flag")); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); -/* -** Required because bestIndex() is called by bestOrClauseIndex() -*/ -static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, - Bitmask, Bitmask, ExprList*, WhereCost*); + /* Begin a loop that will extract all source rows in GROUP BY order. + ** This might involve two separate loops with an OP_Sort in between, or + ** it might be a single loop that uses an index to extract information + ** in the right order to begin with. + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); + if( pWInfo==0 ) goto select_end; + if( pGroupBy==0 ){ + /* The optimizer is able to deliver rows in group by order so + ** we do not have to sort. The OP_OpenEphemeral table will be + ** cancelled later because we still need to use the pKeyInfo + */ + pGroupBy = p->pGroupBy; + groupBySort = 0; + }else{ + /* Rows are coming out in undetermined order. We have to push + ** each row into a sorting index, terminate the first loop, + ** then loop over the sorting index in order to get the output + ** in sorted order + */ + int regBase; + int regRecord; + int nCol; + int nGroupBy; -/* -** This routine attempts to find an scanning strategy that can be used -** to optimize an 'OR' expression that is part of a WHERE clause. -** -** The table associated with FROM clause term pSrc may be either a -** regular B-Tree table or a virtual table. -*/ -static void bestOrClauseIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ - WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ + explainTempTable(pParse, + isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); - /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses - ** are used */ - if( pSrc->notIndexed || pSrc->pIndex!=0 ){ - return; - } + groupBySort = 1; + nGroupBy = pGroupBy->nExpr; + nCol = nGroupBy + 1; + j = nGroupBy+1; + for(i=0; i=j ){ + nCol++; + j++; + } + } + regBase = sqlite3GetTempRange(pParse, nCol); + sqlite3ExprCacheClear(pParse); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); + j = nGroupBy+1; + for(i=0; iiSorterColumn>=j ){ + int r1 = j + regBase; + int r2; - /* Search the WHERE clause terms for a usable WO_OR term. */ - for(pTerm=pWC->a; pTermeOperator==WO_OR - && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 - && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 - ){ - WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; - WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; - WhereTerm *pOrTerm; - int flags = WHERE_MULTI_OR; - double rTotal = 0; - double nRow = 0; - Bitmask used = 0; + r2 = sqlite3ExprCodeGetColumn(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1); + if( r1!=r2 ){ + sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); + } + j++; + } + } + regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); + sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nCol); + sqlite3WhereEnd(pWInfo); + sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); + VdbeComment((v, "GROUP BY sort")); + sAggInfo.useSortingIdx = 1; + sqlite3ExprCacheClear(pParse); + } - for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) - )); - if( pOrTerm->eOperator==WO_AND ){ - WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); - }else if( pOrTerm->leftCursor==iCur ){ - WhereClause tempWC; - tempWC.pParse = pWC->pParse; - tempWC.pMaskSet = pWC->pMaskSet; - tempWC.op = TK_AND; - tempWC.a = pOrTerm; - tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); + /* Evaluate the current GROUP BY terms and store in b0, b1, b2... + ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth) + ** Then compare the current GROUP BY terms against the GROUP BY terms + ** from the previous row currently stored in a0, a1, a2... + */ + addrTopOfLoop = sqlite3VdbeCurrentAddr(v); + sqlite3ExprCacheClear(pParse); + for(j=0; jnExpr; j++){ + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); }else{ - continue; + sAggInfo.directMode = 1; + sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } - rTotal += sTermCost.rCost; - nRow += sTermCost.plan.nRow; - used |= sTermCost.used; - if( rTotal>=pCost->rCost ) break; } + sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, + (char*)pKeyInfo, P4_KEYINFO); + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); - /* If there is an ORDER BY clause, increase the scan cost to account - ** for the cost of the sort. */ - if( pOrderBy!=0 ){ - WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", - rTotal, rTotal+nRow*estLog(nRow))); - rTotal += nRow*estLog(nRow); - } + /* Generate code that runs whenever the GROUP BY changes. + ** Changes in the GROUP BY are detected by the previous code + ** block. If there were no changes, this block is skipped. + ** + ** This code copies current group by terms in b0,b1,b2,... + ** over to a0,a1,a2. It then calls the output subroutine + ** and resets the aggregate accumulator registers in preparation + ** for the next GROUP BY batch. + */ + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output one row")); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); + VdbeComment((v, "check abort flag")); + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + VdbeComment((v, "reset accumulator")); - /* If the cost of scanning using this OR term for optimization is - ** less than the current cost stored in pCost, replace the contents - ** of pCost. */ - WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); - if( rTotalrCost ){ - pCost->rCost = rTotal; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = flags; - pCost->plan.u.pTerm = pTerm; + /* Update the aggregate accumulators based on the content of + ** the current row + */ + sqlite3VdbeJumpHere(v, j1); + updateAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); + VdbeComment((v, "indicate data in accumulator")); + + /* End of the loop + */ + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); + }else{ + sqlite3WhereEnd(pWInfo); + sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); } - } - } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ -} -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** Return TRUE if the WHERE clause term pTerm is of a form where it -** could be used with an index to access pSrc, assuming an appropriate -** index existed. -*/ -static int termCanDriveIndex( - WhereTerm *pTerm, /* WHERE clause term to check */ - struct SrcList_item *pSrc, /* Table we are trying to access */ - Bitmask notReady /* Tables in outer loops of the join */ -){ - char aff; - if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; - if( (pTerm->prereqRight & notReady)!=0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; - if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; - return 1; -} -#endif - -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** If the query plan for pSrc specified in pCost is a full table scan -** and indexing is allows (if there is no NOT INDEXED clause) and it -** possible to construct a transient index that would perform better -** than a full table scan even when the cost of constructing the index -** is taken into account, then alter the query plan to use the -** transient index. -*/ -static void bestAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereCost *pCost /* Lowest cost query plan */ -){ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ - double costTempIdx; /* per-query cost of the transient index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Table *pTable; /* Table tht might be indexed */ - - if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ - /* Automatic indices are disabled at run-time */ - return; - } - if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ - /* We already have some kind of index in use for this query. */ - return; - } - if( pSrc->notIndexed ){ - /* The NOT INDEXED clause appears in the SQL. */ - return; - } - - assert( pParse->nQueryLoop >= (double)1 ); - pTable = pSrc->pTab; - nTableRow = pTable->nRowEst; - logN = estLog(nTableRow); - costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=pCost->rCost ){ - /* The cost of creating the transient table would be greater than - ** doing the full table scan */ - return; - } - - /* Search for any equality comparison term */ - pWCEnd = &pWC->a[pWC->nTerm]; - for(pTerm=pWC->a; pTermrCost, costTempIdx)); - pCost->rCost = costTempIdx; - pCost->plan.nRow = logN + 1; - pCost->plan.wsFlags = WHERE_TEMP_INDEX; - pCost->used = pTerm->prereqRight; - break; - } - } -} -#else -# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + /* Output the final row of result + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output final row")); + /* Jump over the subroutines + */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX -/* -** Generate code to construct the Index object for an automatic index -** and to set up the WhereLevel object pLevel so that the code generator -** makes use of the automatic index. -*/ -static void constructAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereLevel *pLevel /* Write new index here */ -){ - int nColumn; /* Number of columns in the constructed index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - int nByte; /* Byte of memory needed for pIdx */ - Index *pIdx; /* Object describing the transient index */ - Vdbe *v; /* Prepared statement under construction */ - int regIsInit; /* Register set by initialization */ - int addrInit; /* Address of the initialization bypass jump */ - Table *pTable; /* The table being indexed */ - KeyInfo *pKeyinfo; /* Key information for the index */ - int addrTop; /* Top of the index fill loop */ - int regRecord; /* Register holding an index record */ - int n; /* Column counter */ - int i; /* Loop counter */ - int mxBitCol; /* Maximum column in pSrc->colUsed */ - CollSeq *pColl; /* Collating sequence to on a column */ - Bitmask idxCols; /* Bitmap of columns used for indexing */ - Bitmask extraCols; /* Bitmap of additional columns */ + /* Generate a subroutine that outputs a single row of the result + ** set. This subroutine first looks at the iUseFlag. If iUseFlag + ** is less than or equal to zero, the subroutine is a no-op. If + ** the processing calls for the query to abort, this subroutine + ** increments the iAbortFlag memory location before returning in + ** order to signal the caller to abort. + */ + addrSetAbort = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag); + VdbeComment((v, "set abort flag")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + sqlite3VdbeResolveLabel(v, addrOutputRow); + addrOutputRow = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeComment((v, "Groupby result generator entry point")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + finalizeAggFunctions(pParse, &sAggInfo); + sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, + distinct, pDest, + addrOutputRow+1, addrSetAbort); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + VdbeComment((v, "end groupby result generator")); - /* Generate code to skip over the creation and initialization of the - ** transient index on 2nd and subsequent iterations of the loop. */ - v = pParse->pVdbe; - assert( v!=0 ); - regIsInit = ++pParse->nMem; - addrInit = sqlite3VdbeAddOp1(v, OP_If, regIsInit); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regIsInit); + /* Generate a subroutine that will reset the group-by accumulator + */ + sqlite3VdbeResolveLabel(v, addrReset); + resetAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp1(v, OP_Return, regReset); + + } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ + else { + ExprList *pDel = 0; +#ifndef SQLITE_OMIT_BTREECOUNT + Table *pTab; + if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ + /* If isSimpleCount() returns a pointer to a Table structure, then + ** the SQL statement is of the form: + ** + ** SELECT count(*) FROM + ** + ** where the Table structure returned represents table . + ** + ** This statement is so common that it is optimized specially. The + ** OP_Count instruction is executed either on the intkey table that + ** contains the data for table or on one of its indexes. It + ** is better to execute the op on an index, as indexes are almost + ** always spread across less pages than their corresponding tables. + */ + const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */ + Index *pIdx; /* Iterator variable */ + KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */ + Index *pBest = 0; /* Best index found so far */ + int iRoot = pTab->tnum; /* Root page of scanned b-tree */ - /* Count the number of columns that will be added to the index - ** and used to match WHERE clause constraints */ - nColumn = 0; - pTable = pSrc->pTab; - pWCEnd = &pWC->a[pWC->nTerm]; - idxCols = 0; - for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<0 ); - pLevel->plan.nEq = nColumn; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* Count the number of additional columns needed to create a - ** covering index. A "covering index" is an index that contains all - ** columns that are needed by the query. With a covering index, the - ** original table never needs to be accessed. Automatic indices must - ** be a covering index because the index will not be updated if the - ** original table changes and the index and table cannot both be used - ** if they go out of sync. - */ - extraCols = pSrc->colUsed & (~idxCols | (((Bitmask)1)<<(BMS-1))); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; - testcase( pTable->nCol==BMS-1 ); - testcase( pTable->nCol==BMS-2 ); - for(i=0; icolUsed & (((Bitmask)1)<<(BMS-1)) ){ - nColumn += pTable->nCol - BMS + 1; - } - pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ; + /* Search for the index that has the least amount of columns. If + ** there is such an index, and it has less columns than the table + ** does, then we can assume that it consumes less space on disk and + ** will therefore be cheaper to scan to determine the query result. + ** In this case set iRoot to the root page number of the index b-tree + ** and pKeyInfo to the KeyInfo structure required to navigate the + ** index. + ** + ** (2011-04-15) Do not do a full scan of an unordered index. + ** + ** In practice the KeyInfo structure will not be used. It is only + ** passed to keep OP_OpenRead happy. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->bUnordered==0 && (!pBest || pIdx->nColumnnColumn) ){ + pBest = pIdx; + } + } + if( pBest && pBest->nColumnnCol ){ + iRoot = pBest->tnum; + pKeyInfo = sqlite3IndexKeyinfo(pParse, pBest); + } - /* Construct the Index object to describe this index */ - nByte = sizeof(Index); - nByte += nColumn*sizeof(int); /* Index.aiColumn */ - nByte += nColumn*sizeof(char*); /* Index.azColl */ - nByte += nColumn; /* Index.aSortOrder */ - pIdx = sqlite3DbMallocZero(pParse->db, nByte); - if( pIdx==0 ) return; - pLevel->plan.u.pIdx = pIdx; - pIdx->azColl = (char**)&pIdx[1]; - pIdx->aiColumn = (int*)&pIdx->azColl[nColumn]; - pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nColumn]; - pIdx->zName = "auto-index"; - pIdx->nColumn = nColumn; - pIdx->pTable = pTable; - n = 0; - idxCols = 0; - for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<pExpr; - idxCols |= cMask; - pIdx->aiColumn[n] = pTerm->u.leftColumn; - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - pIdx->azColl[n] = pColl->zName; - n++; + /* Open a read-only cursor, execute the OP_Count, close the cursor. */ + sqlite3VdbeAddOp3(v, OP_OpenRead, iCsr, iRoot, iDb); + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO_HANDOFF); + } + sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); + explainSimpleCount(pParse, pTab, pBest); + }else +#endif /* SQLITE_OMIT_BTREECOUNT */ + { + /* Check if the query is of one of the following forms: + ** + ** SELECT min(x) FROM ... + ** SELECT max(x) FROM ... + ** + ** If it is, then ask the code in where.c to attempt to sort results + ** as if there was an "ORDER ON x" or "ORDER ON x DESC" clause. + ** If where.c is able to produce results sorted in this order, then + ** add vdbe code to break out of the processing loop after the + ** first iteration (since the first iteration of the loop is + ** guaranteed to operate on the row with the minimum or maximum + ** value of x, the only row required). + ** + ** A special flag must be passed to sqlite3WhereBegin() to slightly + ** modify behaviour as follows: + ** + ** + If the query is a "SELECT min(x)", then the loop coded by + ** where.c should not iterate over any values with a NULL value + ** for x. + ** + ** + The optimizer code in where.c (the thing that decides which + ** index or indices to use) should place a different priority on + ** satisfying the 'ORDER BY' clause than it does in other cases. + ** Refer to code and comments in where.c for details. + */ + ExprList *pMinMax = 0; + u8 flag = minMaxQuery(p); + if( flag ){ + assert( !ExprHasProperty(p->pEList->a[0].pExpr, EP_xIsSelect) ); + pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->x.pList,0); + pDel = pMinMax; + if( pMinMax && !db->mallocFailed ){ + pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; + pMinMax->a[0].pExpr->op = TK_COLUMN; + } + } + + /* This case runs if the aggregate has no GROUP BY clause. The + ** processing is much simpler since there is only a single row + ** of output. + */ + resetAccumulator(pParse, &sAggInfo); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); + if( pWInfo==0 ){ + sqlite3ExprListDelete(db, pDel); + goto select_end; + } + updateAccumulator(pParse, &sAggInfo); + if( !pMinMax && flag ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); + VdbeComment((v, "%s() by index", + (flag==WHERE_ORDERBY_MIN?"min":"max"))); + } + sqlite3WhereEnd(pWInfo); + finalizeAggFunctions(pParse, &sAggInfo); } - } - } - assert( (u32)n==pLevel->plan.nEq ); - /* Add additional columns needed to make the automatic index into - ** a covering index */ - for(i=0; iaiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; - n++; - } - } - if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ - for(i=BMS-1; inCol; i++){ - pIdx->aiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; - n++; + pOrderBy = 0; + sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, + pDest, addrEnd, addrEnd); + sqlite3ExprListDelete(db, pDel); } - } - assert( n==nColumn ); - - /* Create the automatic index */ - pKeyinfo = sqlite3IndexKeyinfo(pParse, pIdx); - assert( pLevel->iIdxCur>=0 ); - sqlite3VdbeAddOp4(v, OP_OpenAutoindex, pLevel->iIdxCur, nColumn+1, 0, - (char*)pKeyinfo, P4_KEYINFO_HANDOFF); - VdbeComment((v, "for %s", pTable->zName)); - - /* Fill the automatic index with content */ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); - regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); - sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); - sqlite3VdbeJumpHere(v, addrTop); - sqlite3ReleaseTempReg(pParse, regRecord); - - /* Jump here when skipping the initialization */ - sqlite3VdbeJumpHere(v, addrInit); -} -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Allocate and populate an sqlite3_index_info structure. It is the -** responsibility of the caller to eventually release the structure -** by passing the pointer returned by this function to sqlite3_free(). -*/ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - struct SrcList_item *pSrc, - ExprList *pOrderBy -){ - int i, j; - int nTerm; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_orderby *pIdxOrderBy; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int nOrderBy; - sqlite3_index_info *pIdxInfo; - - WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName)); + sqlite3VdbeResolveLabel(v, addrEnd); + + } /* endif aggregate query */ - /* Count the number of possible WHERE clause constraints referring - ** to this virtual table */ - for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; - nTerm++; + if( distinct>=0 ){ + explainTempTable(pParse, "DISTINCT"); } - /* If the ORDER BY clause contains only columns in the current - ** virtual table then allocate space for the aOrderBy part of - ** the sqlite3_index_info structure. + /* If there is an ORDER BY clause, then we need to sort the results + ** and send them to the callback one by one. */ - nOrderBy = 0; if( pOrderBy ){ - for(i=0; inExpr; i++){ - Expr *pExpr = pOrderBy->a[i].pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; - } - if( i==pOrderBy->nExpr ){ - nOrderBy = pOrderBy->nExpr; - } + explainTempTable(pParse, "ORDER BY"); + generateSortTail(pParse, p, v, pEList->nExpr, pDest); } - /* Allocate the sqlite3_index_info structure + /* Jump here to skip this query */ - pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) - + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy ); - if( pIdxInfo==0 ){ - sqlite3ErrorMsg(pParse, "out of memory"); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - return 0; - } + sqlite3VdbeResolveLabel(v, iEnd); - /* Initialize the structure. The sqlite3_index_info structure contains - ** many fields that are declared "const" to prevent xBestIndex from - ** changing them. We have to do some funky casting in order to - ** initialize those fields. + /* The SELECT was successfully coded. Set the return code to 0 + ** to indicate no errors. */ - pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; - pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; - pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - *(int*)&pIdxInfo->nConstraint = nTerm; - *(int*)&pIdxInfo->nOrderBy = nOrderBy; - *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; - *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; - *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = - pUsage; + rc = 0; - for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; - pIdxCons[j].iColumn = pTerm->u.leftColumn; - pIdxCons[j].iTermOffset = i; - pIdxCons[j].op = (u8)pTerm->eOperator; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); - j++; - } - for(i=0; ia[i].pExpr; - pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + /* Control jumps to here if an error is encountered above, or upon + ** successful coding of the SELECT. + */ +select_end: + explainSetInteger(pParse->iSelectId, iRestoreSelectId); + + /* Identify column names if results of the SELECT are to be output. + */ + if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ + generateColumnNames(pParse, pTabList, pEList); } - return pIdxInfo; + sqlite3DbFree(db, sAggInfo.aCol); + sqlite3DbFree(db, sAggInfo.aFunc); + return rc; } +#if defined(SQLITE_DEBUG) /* -** The table object reference passed as the second argument to this function -** must represent a virtual table. This function invokes the xBestIndex() -** method of the virtual table with the sqlite3_index_info pointer passed -** as the argument. +******************************************************************************* +** The following code is used for testing and debugging only. The code +** that follows does not appear in normal builds. ** -** If an error occurs, pParse is populated with an error message and a -** non-zero value is returned. Otherwise, 0 is returned and the output -** part of the sqlite3_index_info structure is left populated. +** These routines are used to print out the content of all or part of a +** parse structures such as Select or Expr. Such printouts are useful +** for helping to understand what is happening inside the code generator +** during the execution of complex SELECT statements. ** -** Whether or not an error is returned, it is the responsibility of the -** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates -** that this is required. +** These routine are not called anywhere from within the normal +** code base. Then are intended to be called from within the debugger +** or from temporary "printf" statements inserted for debugging. */ -static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ - sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; - int i; - int rc; - - WHERETRACE(("xBestIndex for %s\n", pTab->zName)); - TRACE_IDX_INPUTS(p); - rc = pVtab->pModule->xBestIndex(pVtab, p); - TRACE_IDX_OUTPUTS(p); - - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ){ - pParse->db->mallocFailed = 1; - }else if( !pVtab->zErrMsg ){ - sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); - }else{ - sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); - } +SQLITE_PRIVATE void sqlite3PrintExpr(Expr *p){ + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + sqlite3DebugPrintf("(%s", p->u.zToken); + }else{ + sqlite3DebugPrintf("(%d", p->op); } - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; - - for(i=0; inConstraint; i++){ - if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ - sqlite3ErrorMsg(pParse, - "table %s: xBestIndex returned an invalid plan", pTab->zName); + if( p->pLeft ){ + sqlite3DebugPrintf(" "); + sqlite3PrintExpr(p->pLeft); + } + if( p->pRight ){ + sqlite3DebugPrintf(" "); + sqlite3PrintExpr(p->pRight); + } + sqlite3DebugPrintf(")"); +} +SQLITE_PRIVATE void sqlite3PrintExprList(ExprList *pList){ + int i; + for(i=0; inExpr; i++){ + sqlite3PrintExpr(pList->a[i].pExpr); + if( inExpr-1 ){ + sqlite3DebugPrintf(", "); } } - - return pParse->nErr; } +SQLITE_PRIVATE void sqlite3PrintSelect(Select *p, int indent){ + sqlite3DebugPrintf("%*sSELECT(%p) ", indent, "", p); + sqlite3PrintExprList(p->pEList); + sqlite3DebugPrintf("\n"); + if( p->pSrc ){ + char *zPrefix; + int i; + zPrefix = "FROM"; + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + sqlite3DebugPrintf("%*s ", indent+6, zPrefix); + zPrefix = ""; + if( pItem->pSelect ){ + sqlite3DebugPrintf("(\n"); + sqlite3PrintSelect(pItem->pSelect, indent+10); + sqlite3DebugPrintf("%*s)", indent+8, ""); + }else if( pItem->zName ){ + sqlite3DebugPrintf("%s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3DebugPrintf("(table: %s)", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3DebugPrintf(" AS %s", pItem->zAlias); + } + if( ipSrc->nSrc-1 ){ + sqlite3DebugPrintf(","); + } + sqlite3DebugPrintf("\n"); + } + } + if( p->pWhere ){ + sqlite3DebugPrintf("%*s WHERE ", indent, ""); + sqlite3PrintExpr(p->pWhere); + sqlite3DebugPrintf("\n"); + } + if( p->pGroupBy ){ + sqlite3DebugPrintf("%*s GROUP BY ", indent, ""); + sqlite3PrintExprList(p->pGroupBy); + sqlite3DebugPrintf("\n"); + } + if( p->pHaving ){ + sqlite3DebugPrintf("%*s HAVING ", indent, ""); + sqlite3PrintExpr(p->pHaving); + sqlite3DebugPrintf("\n"); + } + if( p->pOrderBy ){ + sqlite3DebugPrintf("%*s ORDER BY ", indent, ""); + sqlite3PrintExprList(p->pOrderBy); + sqlite3DebugPrintf("\n"); + } +} +/* End of the structure debug printing code +*****************************************************************************/ +#endif /* defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ - +/************** End of select.c **********************************************/ +/************** Begin file table.c *******************************************/ /* -** Compute the best index for a virtual table. +** 2001 September 15 ** -** The best index is computed by the xBestIndex method of the virtual -** table module. This routine is really just a wrapper that sets up -** the sqlite3_index_info structure that is used to communicate with -** xBestIndex. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** In a join, this routine might be called multiple times for the -** same virtual table. The sqlite3_index_info structure is created -** and initialized on the first invocation and reused on all subsequent -** invocations. The sqlite3_index_info structure is also used when -** code is generated to access the virtual table. The whereInfoDelete() -** routine takes care of freeing the sqlite3_index_info structure after -** everybody has finished with it. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the sqlite3_get_table() and sqlite3_free_table() +** interface routines. These are just wrappers around the main +** interface routine of sqlite3_exec(). +** +** These routines are in a separate files so that they will not be linked +** if they are not used. */ -static void bestVirtualIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for index */ - Bitmask notValid, /* Cursors not valid for any purpose */ - ExprList *pOrderBy, /* The order by clause */ - WhereCost *pCost, /* Lowest cost query plan */ - sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ -){ - Table *pTab = pSrc->pTab; - sqlite3_index_info *pIdxInfo; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int i, j; - int nOrderBy; - double rCost; - - /* Make sure wsFlags is initialized to some sane value. Otherwise, if the - ** malloc in allocateIndexInfo() fails and this function returns leaving - ** wsFlags in an uninitialized state, the caller may behave unpredictably. - */ - memset(pCost, 0, sizeof(*pCost)); - pCost->plan.wsFlags = WHERE_VIRTUALTABLE; - /* If the sqlite3_index_info structure has not been previously - ** allocated and initialized, then allocate and initialize it now. - */ - pIdxInfo = *ppIdxInfo; - if( pIdxInfo==0 ){ - *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); - } - if( pIdxInfo==0 ){ - return; - } +#ifndef SQLITE_OMIT_GET_TABLE - /* At this point, the sqlite3_index_info structure that pIdxInfo points - ** to will have been initialized, either during the current invocation or - ** during some prior invocation. Now we just have to customize the - ** details of pIdxInfo for the current invocation and pass it to - ** xBestIndex. - */ +/* +** This structure is used to pass data from sqlite3_get_table() through +** to the callback function is uses to build the result. +*/ +typedef struct TabResult { + char **azResult; /* Accumulated output */ + char *zErrMsg; /* Error message text, if an error occurs */ + int nAlloc; /* Slots allocated for azResult[] */ + int nRow; /* Number of rows in the result */ + int nColumn; /* Number of columns in the result */ + int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + int rc; /* Return code from sqlite3_exec() */ +} TabResult; - /* The module name must be defined. Also, by this point there must - ** be a pointer to an sqlite3_vtab structure. Otherwise - ** sqlite3ViewGetColumnNames() would have picked up the error. - */ - assert( pTab->azModuleArg && pTab->azModuleArg[0] ); - assert( sqlite3GetVTable(pParse->db, pTab) ); +/* +** This routine is called once for each row in the result table. Its job +** is to fill in the TabResult structure appropriately, allocating new +** memory as necessary. +*/ +static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ + TabResult *p = (TabResult*)pArg; /* Result accumulator */ + int need; /* Slots needed in p->azResult[] */ + int i; /* Loop counter */ + char *z; /* A single column of result */ - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. + /* Make sure there is enough space in p->azResult to hold everything + ** we need to remember from this invocation of the callback. */ - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; inConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } - - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; + if( p->nRow==0 && argv!=0 ){ + need = nCol*2; + }else{ + need = nCol; } - - pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; inConstraint; i++){ - if( pUsage[i].argvIndex>0 ){ - pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; - } + if( p->nData + need > p->nAlloc ){ + char **azNew; + p->nAlloc = p->nAlloc*2 + need; + azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); + if( azNew==0 ) goto malloc_failed; + p->azResult = azNew; } - /* If there is an ORDER BY clause, and the selected virtual table index - ** does not satisfy it, increase the cost of the scan accordingly. This - ** matches the processing for non-virtual tables in bestBtreeIndex(). + /* If this is the first row, then generate an extra row containing + ** the names of all columns. */ - rCost = pIdxInfo->estimatedCost; - if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ - rCost += estLog(rCost)*rCost; + if( p->nRow==0 ){ + p->nColumn = nCol; + for(i=0; iazResult[p->nData++] = z; + } + }else if( p->nColumn!=nCol ){ + sqlite3_free(p->zErrMsg); + p->zErrMsg = sqlite3_mprintf( + "sqlite3_get_table() called with two or more incompatible queries" + ); + p->rc = SQLITE_ERROR; + return 1; } - /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the - ** inital value of lowestCost in this loop. If it is, then the - ** (costrCost = (SQLITE_BIG_DBL/((double)2)); - }else{ - pCost->rCost = rCost; - } - pCost->plan.u.pVtabIdx = pIdxInfo; - if( pIdxInfo->orderByConsumed ){ - pCost->plan.wsFlags |= WHERE_ORDERBY; + if( argv!=0 ){ + for(i=0; iazResult[p->nData++] = z; + } + p->nRow++; } - pCost->plan.nEq = 0; - pIdxInfo->nOrderBy = nOrderBy; + return 0; - /* Try to find a more efficient access pattern by using multiple indexes - ** to optimize an OR expression within the WHERE clause. - */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); +malloc_failed: + p->rc = SQLITE_NOMEM; + return 1; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ /* -** Argument pIdx is a pointer to an index structure that has an array of -** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column -** stored in Index.aSample. The domain of values stored in said column -** may be thought of as divided into (SQLITE_INDEX_SAMPLES+1) regions. -** Region 0 contains all values smaller than the first sample value. Region -** 1 contains values larger than or equal to the value of the first sample, -** but smaller than the value of the second. And so on. +** Query the database. But instead of invoking a callback for each row, +** malloc() for space to hold the result and return the entire results +** at the conclusion of the call. ** -** If successful, this function determines which of the regions value -** pVal lies in, sets *piRegion to the region index (a value between 0 -** and SQLITE_INDEX_SAMPLES+1, inclusive) and returns SQLITE_OK. -** Or, if an OOM occurs while converting text values between encodings, -** SQLITE_NOMEM is returned and *piRegion is undefined. +** The result that is written to ***pazResult is held in memory obtained +** from malloc(). But the caller cannot free this memory directly. +** Instead, the entire table should be passed to sqlite3_free_table() when +** the calling procedure is finished using it. */ -#ifdef SQLITE_ENABLE_STAT2 -static int whereRangeRegion( - Parse *pParse, /* Database connection */ - Index *pIdx, /* Index to consider domain of */ - sqlite3_value *pVal, /* Value to consider */ - int *piRegion /* OUT: Region of domain in which value lies */ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + char ***pazResult, /* Write the result table here */ + int *pnRow, /* Write the number of rows in the result here */ + int *pnColumn, /* Write the number of columns of result here */ + char **pzErrMsg /* Write error messages here */ ){ - if( ALWAYS(pVal) ){ - IndexSample *aSample = pIdx->aSample; - int i = 0; - int eType = sqlite3_value_type(pVal); - - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - double r = sqlite3_value_double(pVal); - for(i=0; i=SQLITE_TEXT || aSample[i].u.r>r ) break; - } - }else{ - sqlite3 *db = pParse->db; - CollSeq *pColl; - const u8 *z; - int n; - - /* pVal comes from sqlite3ValueFromExpr() so the type cannot be NULL */ - assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); - - if( eType==SQLITE_BLOB ){ - z = (const u8 *)sqlite3_value_blob(pVal); - pColl = db->pDfltColl; - assert( pColl->enc==SQLITE_UTF8 ); - }else{ - pColl = sqlite3GetCollSeq(db, SQLITE_UTF8, 0, *pIdx->azColl); - if( pColl==0 ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", - *pIdx->azColl); - return SQLITE_ERROR; - } - z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); - if( !z ){ - return SQLITE_NOMEM; - } - assert( z && pColl && pColl->xCmp ); - } - n = sqlite3ValueBytes(pVal, pColl->enc); + int rc; + TabResult res; - for(i=0; ienc!=SQLITE_UTF8 ){ - int nSample; - char *zSample = sqlite3Utf8to16( - db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample - ); - if( !zSample ){ - assert( db->mallocFailed ); - return SQLITE_NOMEM; - } - r = pColl->xCmp(pColl->pUser, nSample, zSample, n, z); - sqlite3DbFree(db, zSample); - }else -#endif - { - r = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); - } - if( r>0 ) break; + *pazResult = 0; + if( pnColumn ) *pnColumn = 0; + if( pnRow ) *pnRow = 0; + if( pzErrMsg ) *pzErrMsg = 0; + res.zErrMsg = 0; + res.nRow = 0; + res.nColumn = 0; + res.nData = 1; + res.nAlloc = 20; + res.rc = SQLITE_OK; + res.azResult = sqlite3_malloc(sizeof(char*)*res.nAlloc ); + if( res.azResult==0 ){ + db->errCode = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + res.azResult[0] = 0; + rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); + assert( sizeof(res.azResult[0])>= sizeof(res.nData) ); + res.azResult[0] = SQLITE_INT_TO_PTR(res.nData); + if( (rc&0xff)==SQLITE_ABORT ){ + sqlite3_free_table(&res.azResult[1]); + if( res.zErrMsg ){ + if( pzErrMsg ){ + sqlite3_free(*pzErrMsg); + *pzErrMsg = sqlite3_mprintf("%s",res.zErrMsg); } + sqlite3_free(res.zErrMsg); } - - assert( i>=0 && i<=SQLITE_INDEX_SAMPLES ); - *piRegion = i; + db->errCode = res.rc; /* Assume 32-bit assignment is atomic */ + return res.rc; } - return SQLITE_OK; + sqlite3_free(res.zErrMsg); + if( rc!=SQLITE_OK ){ + sqlite3_free_table(&res.azResult[1]); + return rc; + } + if( res.nAlloc>res.nData ){ + char **azNew; + azNew = sqlite3_realloc( res.azResult, sizeof(char*)*res.nData ); + if( azNew==0 ){ + sqlite3_free_table(&res.azResult[1]); + db->errCode = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + res.azResult = azNew; + } + *pazResult = &res.azResult[1]; + if( pnColumn ) *pnColumn = res.nColumn; + if( pnRow ) *pnRow = res.nRow; + return rc; } -#endif /* #ifdef SQLITE_ENABLE_STAT2 */ /* -** If expression pExpr represents a literal value, set *pp to point to -** an sqlite3_value structure containing the same value, with affinity -** aff applied to it, before returning. It is the responsibility of the -** caller to eventually release this structure by passing it to -** sqlite3ValueFree(). -** -** If the current parse is a recompile (sqlite3Reprepare()) and pExpr -** is an SQL variable that currently has a non-NULL value bound to it, -** create an sqlite3_value structure containing this value, again with -** affinity aff applied to it, instead. -** -** If neither of the above apply, set *pp to NULL. -** -** If an error occurs, return an error code. Otherwise, SQLITE_OK. +** This routine frees the space the sqlite3_get_table() malloced. */ -#ifdef SQLITE_ENABLE_STAT2 -static int valueFromExpr( - Parse *pParse, - Expr *pExpr, - u8 aff, - sqlite3_value **pp +SQLITE_API void sqlite3_free_table( + char **azResult /* Result returned from from sqlite3_get_table() */ ){ - /* The evalConstExpr() function will have already converted any TK_VARIABLE - ** expression involved in an comparison into a TK_REGISTER. */ - assert( pExpr->op!=TK_VARIABLE ); - if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){ - int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ - *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); - return SQLITE_OK; + if( azResult ){ + int i, n; + azResult--; + assert( azResult!=0 ); + n = SQLITE_PTR_TO_INT(azResult[0]); + for(i=1; idb, pExpr, SQLITE_UTF8, aff, pp); } -#endif +#endif /* SQLITE_OMIT_GET_TABLE */ + +/************** End of table.c ***********************************************/ +/************** Begin file trigger.c *****************************************/ /* -** This function is used to estimate the number of rows that will be visited -** by scanning an index for a range of values. The range may have an upper -** bound, a lower bound, or both. The WHERE clause terms that set the upper -** and lower bounds are represented by pLower and pUpper respectively. For -** example, assuming that index p is on t1(a): -** -** ... FROM t1 WHERE a > ? AND a < ? ... -** |_____| |_____| -** | | -** pLower pUpper -** -** If either of the upper or lower bound is not present, then NULL is passed in -** place of the corresponding WhereTerm. -** -** The nEq parameter is passed the index of the index column subject to the -** range constraint. Or, equivalently, the number of equality constraints -** optimized by the proposed index scan. For example, assuming index p is -** on t1(a, b), and the SQL query is: -** -** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... ** -** then nEq should be passed the value 1 (as the range restricted column, -** b, is the second left-most column of the index). Or, if the query is: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** ... FROM t1 WHERE a > ? AND a < ? ... +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** then nEq should be passed 0. -** -** The returned value is an integer between 1 and 100, inclusive. A return -** value of 1 indicates that the proposed range scan is expected to visit -** approximately 1/100th (1%) of the rows selected by the nEq equality -** constraints (if any). A return value of 100 indicates that it is expected -** that the range scan will visit every row (100%) selected by the equality -** constraints. -** -** In the absence of sqlite_stat2 ANALYZE data, each range inequality -** reduces the search space by 2/3rds. Hence a single constraint (x>?) -** results in a return of 33 and a range constraint (x>? AND xaCol[] of the range-compared column */ - WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - int *piEst /* OUT: Return value */ -){ - int rc = SQLITE_OK; - -#ifdef SQLITE_ENABLE_STAT2 - - if( nEq==0 && p->aSample ){ - sqlite3_value *pLowerVal = 0; - sqlite3_value *pUpperVal = 0; - int iEst; - int iLower = 0; - int iUpper = SQLITE_INDEX_SAMPLES; - u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; - - if( pLower ){ - Expr *pExpr = pLower->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pLowerVal); - } - if( rc==SQLITE_OK && pUpper ){ - Expr *pExpr = pUpper->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pUpperVal); - } - if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){ - sqlite3ValueFree(pLowerVal); - sqlite3ValueFree(pUpperVal); - goto range_est_fallback; - }else if( pLowerVal==0 ){ - rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper); - if( pLower ) iLower = iUpper/2; - }else if( pUpperVal==0 ){ - rc = whereRangeRegion(pParse, p, pLowerVal, &iLower); - if( pUpper ) iUpper = (iLower + SQLITE_INDEX_SAMPLES + 1)/2; - }else{ - rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper); - if( rc==SQLITE_OK ){ - rc = whereRangeRegion(pParse, p, pLowerVal, &iLower); - } - } +#ifndef SQLITE_OMIT_TRIGGER +/* +** Delete a linked list of TriggerStep structures. +*/ +SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ + while( pTriggerStep ){ + TriggerStep * pTmp = pTriggerStep; + pTriggerStep = pTriggerStep->pNext; - iEst = iUpper - iLower; - testcase( iEst==SQLITE_INDEX_SAMPLES ); - assert( iEst<=SQLITE_INDEX_SAMPLES ); - if( iEst<1 ){ - iEst = 1; - } + sqlite3ExprDelete(db, pTmp->pWhere); + sqlite3ExprListDelete(db, pTmp->pExprList); + sqlite3SelectDelete(db, pTmp->pSelect); + sqlite3IdListDelete(db, pTmp->pIdList); - sqlite3ValueFree(pLowerVal); - sqlite3ValueFree(pUpperVal); - *piEst = (iEst * 100)/SQLITE_INDEX_SAMPLES; - return rc; - } -range_est_fallback: -#else - UNUSED_PARAMETER(pParse); - UNUSED_PARAMETER(p); - UNUSED_PARAMETER(nEq); -#endif - assert( pLower || pUpper ); - if( pLower && pUpper ){ - *piEst = 11; - }else{ - *piEst = 33; + sqlite3DbFree(db, pTmp); } - return rc; } - /* -** Find the query plan for accessing a particular table. Write the -** best query plan and its cost into the WhereCost object supplied as the -** last parameter. -** -** The lowest cost plan wins. The cost is an estimate of the amount of -** CPU and disk I/O need to process the request using the selected plan. -** Factors that influence cost include: -** -** * The estimated number of rows that will be retrieved. (The -** fewer the better.) -** -** * Whether or not sorting must occur. -** -** * Whether or not there must be separate lookups in the -** index and in the main table. +** Given table pTab, return a list of all the triggers attached to +** the table. The list is connected by Trigger.pNext pointers. ** -** If there was an INDEXED BY clause (pSrc->pIndex) attached to the table in -** the SQL statement, then this function only considers plans using the -** named index. If no such plan is found, then the returned cost is -** SQLITE_BIG_DBL. If a plan is found that uses the named index, -** then the cost is calculated in the usual way. +** All of the triggers on pTab that are in the same database as pTab +** are already attached to pTab->pTrigger. But there might be additional +** triggers on pTab in the TEMP schema. This routine prepends all +** TEMP triggers on pTab to the beginning of the pTab->pTrigger list +** and returns the combined list. ** -** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table -** in the SELECT statement, then no indexes are considered. However, the -** selected plan may still take advantage of the tables built-in rowid -** index. +** To state it another way: This routine returns a list of all triggers +** that fire off of pTab. The list will include any TEMP triggers on +** pTab as well as the triggers lised in pTab->pTrigger. */ -static void bestBtreeIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ - int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - Index *pProbe; /* An index we are evaluating */ - Index *pIdx; /* Copy of pProbe, or zero for IPK index */ - int eqTermMask; /* Current mask of valid equality operators */ - int idxEqTermMask; /* Index mask of valid equality operators */ - Index sPk; /* A fake index object for the primary key */ - unsigned int aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ - int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ - int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ - - /* Initialize the cost to a worst-case value */ - memset(pCost, 0, sizeof(*pCost)); - pCost->rCost = SQLITE_BIG_DBL; +SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ + Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; + Trigger *pList = 0; /* List of triggers to return */ - /* If the pSrc table is the right table of a LEFT JOIN then we may not - ** use an index to satisfy IS NULL constraints on that table. This is - ** because columns might end up being NULL if the table does not match - - ** a circumstance which the index cannot help us discover. Ticket #2177. - */ - if( pSrc->jointype & JT_LEFT ){ - idxEqTermMask = WO_EQ|WO_IN; - }else{ - idxEqTermMask = WO_EQ|WO_IN|WO_ISNULL; + if( pParse->disableTriggers ){ + return 0; } - if( pSrc->pIndex ){ - /* An INDEXED BY clause specifies a particular index to use */ - pIdx = pProbe = pSrc->pIndex; - wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); - eqTermMask = idxEqTermMask; - }else{ - /* There is no INDEXED BY clause. Create a fake Index object to - ** represent the primary key */ - Index *pFirst; /* Any other index on the table */ - memset(&sPk, 0, sizeof(Index)); - sPk.nColumn = 1; - sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; - sPk.onError = OE_Replace; - sPk.pTable = pSrc->pTab; - aiRowEstPk[0] = pSrc->pTab->nRowEst; - aiRowEstPk[1] = 1; - pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ - sPk.pNext = pFirst; + if( pTmpSchema!=pTab->pSchema ){ + HashElem *p; + assert( sqlite3SchemaMutexHeld(pParse->db, 0, pTmpSchema) ); + for(p=sqliteHashFirst(&pTmpSchema->trigHash); p; p=sqliteHashNext(p)){ + Trigger *pTrig = (Trigger *)sqliteHashData(p); + if( pTrig->pTabSchema==pTab->pSchema + && 0==sqlite3StrICmp(pTrig->table, pTab->zName) + ){ + pTrig->pNext = (pList ? pList : pTab->pTrigger); + pList = pTrig; + } } - pProbe = &sPk; - wsFlagMask = ~( - WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE - ); - eqTermMask = WO_EQ|WO_IN; - pIdx = 0; } - /* Loop over all indices looking for the best one to use - */ - for(; pProbe; pIdx=pProbe=pProbe->pNext){ - const unsigned int * const aiRowEst = pProbe->aiRowEst; - double cost; /* Cost of using pProbe */ - double nRow; /* Estimated number of rows in result set */ - int rev; /* True to scan in reverse order */ - int wsFlags = 0; - Bitmask used = 0; + return (pList ? pList : pTab->pTrigger); +} - /* The following variables are populated based on the properties of - ** scan being evaluated. They are then used to determine the expected - ** cost and number of rows returned. - ** - ** nEq: - ** Number of equality terms that can be implemented using the index. - ** - ** nInMul: - ** The "in-multiplier". This is an estimate of how many seek operations - ** SQLite must perform on the index in question. For example, if the - ** WHERE clause is: - ** - ** WHERE a IN (1, 2, 3) AND b IN (4, 5, 6) - ** - ** SQLite must perform 9 lookups on an index on (a, b), so nInMul is - ** set to 9. Given the same schema and either of the following WHERE - ** clauses: - ** - ** WHERE a = 1 - ** WHERE a >= 2 - ** - ** nInMul is set to 1. - ** - ** If there exists a WHERE term of the form "x IN (SELECT ...)", then - ** the sub-select is assumed to return 25 rows for the purposes of - ** determining nInMul. - ** - ** bInEst: - ** Set to true if there was at least one "x IN (SELECT ...)" term used - ** in determining the value of nInMul. - ** - ** estBound: - ** An estimate on the amount of the table that must be searched. A - ** value of 100 means the entire table is searched. Range constraints - ** might reduce this to a value less than 100 to indicate that only - ** a fraction of the table needs searching. In the absence of - ** sqlite_stat2 ANALYZE data, a single inequality reduces the search - ** space to 1/3rd its original size. So an x>? constraint reduces - ** estBound to 33. Two constraints (x>? AND xpNewTrigger. After the trigger actions have been parsed, the +** sqlite3FinishTrigger() function is called to complete the trigger +** construction process. +*/ +SQLITE_PRIVATE void sqlite3BeginTrigger( + Parse *pParse, /* The parse context of the CREATE TRIGGER statement */ + Token *pName1, /* The name of the trigger */ + Token *pName2, /* The name of the trigger */ + int tr_tm, /* One of TK_BEFORE, TK_AFTER, TK_INSTEAD */ + int op, /* One of TK_INSERT, TK_UPDATE, TK_DELETE */ + IdList *pColumns, /* column list if this is an UPDATE OF trigger */ + SrcList *pTableName,/* The name of the table/view the trigger applies to */ + Expr *pWhen, /* WHEN clause */ + int isTemp, /* True if the TEMPORARY keyword is present */ + int noErr /* Suppress errors if the trigger already exists */ +){ + Trigger *pTrigger = 0; /* The new trigger */ + Table *pTab; /* Table that the trigger fires off of */ + char *zName = 0; /* Name of the trigger */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* The database to store the trigger in */ + Token *pName; /* The unqualified db name */ + DbFixer sFix; /* State vector for the DB fixer */ + int iTabDb; /* Index of the database holding pTab */ - /* Determine the values of nEq and nInMul */ - for(nEq=0; nEqnColumn; nEq++){ - int j = pProbe->aiColumn[nEq]; - pTerm = findTerm(pWC, iCur, j, notReady, eqTermMask, pIdx); - if( pTerm==0 ) break; - wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); - if( pTerm->eOperator & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - nInMul *= 25; - bInEst = 1; - }else if( ALWAYS(pExpr->x.pList) ){ - nInMul *= pExpr->x.pList->nExpr + 1; - } - }else if( pTerm->eOperator & WO_ISNULL ){ - wsFlags |= WHERE_COLUMN_NULL; - } - used |= pTerm->prereqRight; + assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ + assert( pName2!=0 ); + assert( op==TK_INSERT || op==TK_UPDATE || op==TK_DELETE ); + assert( op>0 && op<0xff ); + if( isTemp ){ + /* If TEMP was specified, then the trigger name may not be qualified. */ + if( pName2->n>0 ){ + sqlite3ErrorMsg(pParse, "temporary trigger may not have qualified name"); + goto trigger_cleanup; } - - /* Determine the value of estBound. */ - if( nEqnColumn ){ - int j = pProbe->aiColumn[nEq]; - if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ - WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); - WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &estBound); - if( pTop ){ - nBound = 1; - wsFlags |= WHERE_TOP_LIMIT; - used |= pTop->prereqRight; - } - if( pBtm ){ - nBound++; - wsFlags |= WHERE_BTM_LIMIT; - used |= pBtm->prereqRight; - } - wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); - } - }else if( pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; - } + iDb = 1; + pName = pName1; + }else{ + /* Figure out the db that the the trigger will be created in */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ){ + goto trigger_cleanup; } + } - /* If there is an ORDER BY clause and the index being considered will - ** naturally scan rows in the required order, set the appropriate flags - ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index - ** will scan rows in a different order, set the bSort variable. */ - if( pOrderBy ){ - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 - && isSortingIndex(pParse,pWC->pMaskSet,pProbe,iCur,pOrderBy,nEq,&rev) - ){ - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - }else{ - bSort = 1; - } - } + /* If the trigger name was unqualified, and the table is a temp table, + ** then set iDb to 1 to create the trigger in the temporary database. + ** If sqlite3SrcListLookup() returns 0, indicating the table does not + ** exist, the error is caught by the block below. + */ + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + pTab = sqlite3SrcListLookup(pParse, pTableName); + if( db->init.busy==0 && pName2->n==0 && pTab + && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } - /* If currently calculating the cost of using an index (not the IPK - ** index), determine if all required column data may be obtained without - ** using the main table (i.e. if the index is a covering - ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** wsFlags. Otherwise, set the bLookup variable to true. */ - if( pIdx && wsFlags ){ - Bitmask m = pSrc->colUsed; - int j; - for(j=0; jnColumn; j++){ - int x = pIdx->aiColumn[j]; - if( xmallocFailed ) goto trigger_cleanup; + assert( pTableName->nSrc==1 ); + if( sqlite3FixInit(&sFix, pParse, iDb, "trigger", pName) && + sqlite3FixSrcList(&sFix, pTableName) ){ + goto trigger_cleanup; + } + pTab = sqlite3SrcListLookup(pParse, pTableName); + if( !pTab ){ + /* The table does not exist. */ + if( db->init.iDb==1 ){ + /* Ticket #3810. + ** Normally, whenever a table is dropped, all associated triggers are + ** dropped too. But if a TEMP trigger is created on a non-TEMP table + ** and the table is dropped by a different database connection, the + ** trigger is not visible to the database connection that does the + ** drop so the trigger cannot be dropped. This results in an + ** "orphaned trigger" - a trigger whose associated table is missing. + */ + db->init.orphanTrigger = 1; } + goto trigger_cleanup; + } + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); + goto trigger_cleanup; + } - /* - ** Estimate the number of rows of output. For an IN operator, - ** do not let the estimate exceed half the rows in the table. - */ - nRow = (double)(aiRowEst[nEq] * nInMul); - if( bInEst && nRow*2>aiRowEst[0] ){ - nRow = aiRowEst[0]/2; - nInMul = (int)(nRow / aiRowEst[nEq]); + /* Check that the trigger name is not reserved and that no trigger of the + ** specified name exists */ + zName = sqlite3NameFromToken(db, pName); + if( !zName || SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + goto trigger_cleanup; + } + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), + zName, sqlite3Strlen30(zName)) ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); } + goto trigger_cleanup; + } - /* Assume constant cost to access a row and logarithmic cost to - ** do a binary search. Hence, the initial cost is the number of output - ** rows plus log2(table-size) times the number of binary searches. - */ - cost = nRow + nInMul*estLog(aiRowEst[0]); + /* Do not create a trigger on a system table */ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); + pParse->nErr++; + goto trigger_cleanup; + } - /* Adjust the number of rows and the cost downward to reflect rows - ** that are excluded by range constraints. - */ - nRow = (nRow * (double)estBound) / (double)100; - cost = (cost * (double)estBound) / (double)100; + /* INSTEAD of triggers are only for views and views only support INSTEAD + ** of triggers. + */ + if( pTab->pSelect && tr_tm!=TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", + (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName, 0); + goto trigger_cleanup; + } + if( !pTab->pSelect && tr_tm==TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create INSTEAD OF" + " trigger on table: %S", pTableName, 0); + goto trigger_cleanup; + } + iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); - /* Add in the estimated cost of sorting the result - */ - if( bSort ){ - cost += cost*estLog(cost); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_CREATE_TRIGGER; + const char *zDb = db->aDb[iTabDb].zName; + const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb; + if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; + if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ + goto trigger_cleanup; } - - /* If all information can be taken directly from the index, we avoid - ** doing table lookups. This reduces the cost by half. (Not really - - ** this needs to be fixed.) - */ - if( pIdx && bLookup==0 ){ - cost /= (double)2; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iTabDb),0,zDb)){ + goto trigger_cleanup; } - /**** Cost of using this index has now been computed ****/ + } +#endif - /* If there are additional constraints on this table that cannot - ** be used with the current index, but which might lower the number - ** of output rows, adjust the nRow value accordingly. This only - ** matters if the current index is the least costly, so do not bother - ** with this step if we already know this index will not be chosen. - ** Also, never reduce the output row count below 2 using this step. - ** - ** It is critical that the notValid mask be used here instead of - ** the notReady mask. When computing an "optimal" index, the notReady - ** mask will only have one bit set - the bit for the current table. - ** The notValid mask, on the other hand, always has all bits set for - ** tables that are not in outer loops. If notReady is used here instead - ** of notValid, then a optimal index that depends on inner joins loops - ** might be selected even when there exists an optimal index that has - ** no such dependency. - */ - if( nRow>2 && cost<=pCost->rCost ){ - int k; /* Loop counter */ - int nSkipEq = nEq; /* Number of == constraints to skip */ - int nSkipRange = nBound; /* Number of < constraints to skip */ - Bitmask thisTab; /* Bitmap for pSrc */ + /* INSTEAD OF triggers can only appear on views and BEFORE triggers + ** cannot appear on views. So we might as well translate every + ** INSTEAD OF trigger into a BEFORE trigger. It simplifies code + ** elsewhere. + */ + if (tr_tm == TK_INSTEAD){ + tr_tm = TK_BEFORE; + } - thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ - if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notValid)!=thisTab ) continue; - if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ - if( nSkipEq ){ - /* Ignore the first nEq equality matches since the index - ** has already accounted for these */ - nSkipEq--; - }else{ - /* Assume each additional equality match reduces the result - ** set size by a factor of 10 */ - nRow /= 10; - } - }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ - if( nSkipRange ){ - /* Ignore the first nBound range constraints since the index - ** has already accounted for these */ - nSkipRange--; - }else{ - /* Assume each additional range constraint reduces the result - ** set size by a factor of 3 */ - nRow /= 3; - } - }else{ - /* Any other expression lowers the output row count by half */ - nRow /= 2; - } - } - if( nRow<2 ) nRow = 2; - } + /* Build the Trigger object */ + pTrigger = (Trigger*)sqlite3DbMallocZero(db, sizeof(Trigger)); + if( pTrigger==0 ) goto trigger_cleanup; + pTrigger->zName = zName; + zName = 0; + pTrigger->table = sqlite3DbStrDup(db, pTableName->a[0].zName); + pTrigger->pSchema = db->aDb[iDb].pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pTrigger->op = (u8)op; + pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + pTrigger->pColumns = sqlite3IdListDup(db, pColumns); + assert( pParse->pNewTrigger==0 ); + pParse->pNewTrigger = pTrigger; +trigger_cleanup: + sqlite3DbFree(db, zName); + sqlite3SrcListDelete(db, pTableName); + sqlite3IdListDelete(db, pColumns); + sqlite3ExprDelete(db, pWhen); + if( !pParse->pNewTrigger ){ + sqlite3DeleteTrigger(db, pTrigger); + }else{ + assert( pParse->pNewTrigger==pTrigger ); + } +} - WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d estBound=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx nRow=%.2f cost=%.2f used=0x%llx\n", - pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), - nEq, nInMul, estBound, bSort, bLookup, wsFlags, - notReady, nRow, cost, used - )); +/* +** This routine is called after all of the trigger actions have been parsed +** in order to complete the process of building the trigger. +*/ +SQLITE_PRIVATE void sqlite3FinishTrigger( + Parse *pParse, /* Parser context */ + TriggerStep *pStepList, /* The triggered program */ + Token *pAll /* Token that describes the complete CREATE TRIGGER */ +){ + Trigger *pTrig = pParse->pNewTrigger; /* Trigger being finished */ + char *zName; /* Name of trigger */ + sqlite3 *db = pParse->db; /* The database */ + DbFixer sFix; /* Fixer object */ + int iDb; /* Database containing the trigger */ + Token nameToken; /* Trigger name for error reporting */ - /* If this index is the best we have seen so far, then record this - ** index and its cost in the pCost structure. - */ - if( (!pIdx || wsFlags) - && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) - ){ - pCost->rCost = cost; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = (wsFlags&wsFlagMask); - pCost->plan.nEq = nEq; - pCost->plan.u.pIdx = pIdx; - } + pParse->pNewTrigger = 0; + if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; + zName = pTrig->zName; + iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + pTrig->step_list = pStepList; + while( pStepList ){ + pStepList->pTrig = pTrig; + pStepList = pStepList->pNext; + } + nameToken.z = pTrig->zName; + nameToken.n = sqlite3Strlen30(nameToken.z); + if( sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken) + && sqlite3FixTriggerStep(&sFix, pTrig->step_list) ){ + goto triggerfinish_cleanup; + } - /* If there was an INDEXED BY clause, then only that one index is - ** considered. */ - if( pSrc->pIndex ) break; + /* if we are not initializing, + ** build the sqlite_master entry + */ + if( !db->init.busy ){ + Vdbe *v; + char *z; - /* Reset masks for the next index in the loop */ - wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); - eqTermMask = idxEqTermMask; + /* Make an entry in the sqlite_master table */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto triggerfinish_cleanup; + sqlite3BeginWriteOperation(pParse, 0, iDb); + z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); + sqlite3NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, + pTrig->table, z); + sqlite3DbFree(db, z); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); } - /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag - ** is set, then reverse the order that the index will be scanned - ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that - ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ - pCost->plan.wsFlags |= WHERE_REVERSE; + if( db->init.busy ){ + Trigger *pLink = pTrig; + Hash *pHash = &db->aDb[iDb].pSchema->trigHash; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); + if( pTrig ){ + db->mallocFailed = 1; + }else if( pLink->pSchema==pLink->pTabSchema ){ + Table *pTab; + int n = sqlite3Strlen30(pLink->table); + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); + assert( pTab!=0 ); + pLink->pNext = pTab->pTrigger; + pTab->pTrigger = pLink; + } } - assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); - assert( pCost->plan.u.pIdx==0 || (pCost->plan.wsFlags&WHERE_ROWID_EQ)==0 ); - assert( pSrc->pIndex==0 - || pCost->plan.u.pIdx==0 - || pCost->plan.u.pIdx==pSrc->pIndex - ); - - WHERETRACE(("best index is: %s\n", - ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") - )); - - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); - pCost->plan.wsFlags |= eqTermMask; +triggerfinish_cleanup: + sqlite3DeleteTrigger(db, pTrig); + assert( !pParse->pNewTrigger ); + sqlite3DeleteTriggerStep(db, pStepList); } /* -** Find the query plan for accessing table pSrc->pTab. Write the -** best query plan and its cost into the WhereCost object supplied -** as the last parameter. This function may calculate the cost of -** both real and virtual table scans. +** Turn a SELECT statement (that the pSelect parameter points to) into +** a trigger step. Return a pointer to a TriggerStep structure. +** +** The parser calls this routine when it finds a SELECT statement in +** body of a TRIGGER. */ -static void bestIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pSrc->pTab) ){ - sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); - if( p->needToFreeIdxStr ){ - sqlite3_free(p->idxStr); - } - sqlite3DbFree(pParse->db, p); - }else -#endif - { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){ + TriggerStep *pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); + if( pTriggerStep==0 ) { + sqlite3SelectDelete(db, pSelect); + return 0; } + pTriggerStep->op = TK_SELECT; + pTriggerStep->pSelect = pSelect; + pTriggerStep->orconf = OE_Default; + return pTriggerStep; } /* -** Disable a term in the WHERE clause. Except, do not disable the term -** if it controls a LEFT OUTER JOIN and it did not originate in the ON -** or USING clause of that join. -** -** Consider the term t2.z='ok' in the following queries: -** -** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' -** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' -** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' -** -** The t2.z='ok' is disabled in the in (2) because it originates -** in the ON clause. The term is disabled in (3) because it is not part -** of a LEFT OUTER JOIN. In (1), the term is not disabled. -** -** IMPLEMENTATION-OF: R-24597-58655 No tests are done for terms that are -** completely satisfied by indices. +** Allocate space to hold a new trigger step. The allocated space +** holds both the TriggerStep object and the TriggerStep.target.z string. ** -** Disabling a term causes that term to not be tested in the inner loop -** of the join. Disabling is an optimization. When terms are satisfied -** by indices, we disable them to prevent redundant tests in the inner -** loop. We would get the correct results if nothing were ever disabled, -** but joins might run a little slower. The trick is to disable as much -** as we can without disabling too much. If we disabled in (1), we'd get -** the wrong answer. See ticket #813. +** If an OOM error occurs, NULL is returned and db->mallocFailed is set. */ -static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ - if( pTerm - && (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - ){ - pTerm->wtFlags |= TERM_CODED; - if( pTerm->iParent>=0 ){ - WhereTerm *pOther = &pTerm->pWC->a[pTerm->iParent]; - if( (--pOther->nChild)==0 ){ - disableTerm(pLevel, pOther); - } - } +static TriggerStep *triggerStepAllocate( + sqlite3 *db, /* Database connection */ + u8 op, /* Trigger opcode */ + Token *pName /* The target name */ +){ + TriggerStep *pTriggerStep; + + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n); + if( pTriggerStep ){ + char *z = (char*)&pTriggerStep[1]; + memcpy(z, pName->z, pName->n); + pTriggerStep->target.z = z; + pTriggerStep->target.n = pName->n; + pTriggerStep->op = op; } + return pTriggerStep; } /* -** Code an OP_Affinity opcode to apply the column affinity string zAff -** to the n registers starting at base. -** -** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_NONE, then no code gets generated. +** Build a trigger step out of an INSERT statement. Return a pointer +** to the new trigger step. ** -** This routine makes its own copy of zAff so that the caller is free -** to modify zAff after this routine returns. +** The parser calls this routine when it sees an INSERT inside the +** body of a trigger. */ -static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ - Vdbe *v = pParse->pVdbe; - if( zAff==0 ){ - assert( pParse->db->mallocFailed ); - return; - } - assert( v!=0 ); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( + sqlite3 *db, /* The database connection */ + Token *pTableName, /* Name of the table into which we insert */ + IdList *pColumn, /* List of columns in pTableName to insert into */ + ExprList *pEList, /* The VALUE clause: a list of values to be inserted */ + Select *pSelect, /* A SELECT statement that supplies values */ + u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ +){ + TriggerStep *pTriggerStep; - /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning - ** and end of the affinity string. - */ - while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ - n--; - base++; - zAff++; - } - while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ - n--; - } + assert(pEList == 0 || pSelect == 0); + assert(pEList != 0 || pSelect != 0 || db->mallocFailed); - /* Code the OP_Affinity opcode if there is anything left to do. */ - if( n>0 ){ - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); + pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); + if( pTriggerStep ){ + pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + pTriggerStep->pIdList = pColumn; + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->orconf = orconf; + }else{ + sqlite3IdListDelete(db, pColumn); } -} + sqlite3ExprListDelete(db, pEList); + sqlite3SelectDelete(db, pSelect); + return pTriggerStep; +} /* -** Generate code for a single equality term of the WHERE clause. An equality -** term can be either X=expr or X IN (...). pTerm is the term to be -** coded. -** -** The current value for the constraint is left in register iReg. -** -** For a constraint of the form X=expr, the expression is evaluated and its -** result is left on the stack. For constraints of the form X IN (...) -** this routine sets up a loop that will iterate over all values of X. +** Construct a trigger step that implements an UPDATE statement and return +** a pointer to that trigger step. The parser calls this routine when it +** sees an UPDATE statement inside the body of a CREATE TRIGGER. */ -static int codeEqualityTerm( - Parse *pParse, /* The parsing context */ - WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* When level of the FROM clause we are working on */ - int iTarget /* Attempt to leave results in this register */ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( + sqlite3 *db, /* The database connection */ + Token *pTableName, /* Name of the table to be updated */ + ExprList *pEList, /* The SET clause: list of column and new values */ + Expr *pWhere, /* The WHERE clause */ + u8 orconf /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ ){ - Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; - int iReg; /* Register holding results */ - - assert( iTarget>0 ); - if( pX->op==TK_EQ ){ - iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); - }else if( pX->op==TK_ISNULL ){ - iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); -#ifndef SQLITE_OMIT_SUBQUERY - }else{ - int eType; - int iTab; - struct InLoop *pIn; + TriggerStep *pTriggerStep; - assert( pX->op==TK_IN ); - iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, 0); - iTab = pX->iTable; - sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); - assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - } - pLevel->u.in.nIn++; - pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - pIn += pLevel->u.in.nIn - 1; - pIn->iCur = iTab; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); - }else{ - pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); - } - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); - }else{ - pLevel->u.in.nIn = 0; - } -#endif + pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName); + if( pTriggerStep ){ + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pTriggerStep->orconf = orconf; } - disableTerm(pLevel, pTerm); - return iReg; + sqlite3ExprListDelete(db, pEList); + sqlite3ExprDelete(db, pWhere); + return pTriggerStep; } /* -** Generate code that will evaluate all == and IN constraints for an -** index. -** -** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). -** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 -** The index has as many as three equality constraints, but in this -** example, the third "c" value is an inequality. So only two -** constraints are coded. This routine will generate code to evaluate -** a==5 and b IN (1,2,3). The current values for a and b will be stored -** in consecutive registers and the index of the first register is returned. -** -** In the example above nEq==2. But this subroutine works for any value -** of nEq including 0. If nEq==0, this routine is nearly a no-op. -** The only thing it does is allocate the pLevel->iMem memory cell and -** compute the affinity string. -** -** This routine always allocates at least one memory cell and returns -** the index of that memory cell. The code that -** calls this routine will use that memory cell to store the termination -** key value of the loop. If one or more IN operators appear, then -** this routine allocates an additional nEq memory cells for internal -** use. -** -** Before returning, *pzAff is set to point to a buffer containing a -** copy of the column affinity string of the index allocated using -** sqlite3DbMalloc(). Except, entries in the copy of the string associated -** with equality constraints that use NONE affinity are set to -** SQLITE_AFF_NONE. This is to deal with SQL such as the following: -** -** CREATE TABLE t1(a TEXT PRIMARY KEY, b); -** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; -** -** In the example above, the index on t1(a) has TEXT affinity. But since -** the right hand side of the equality constraint (t2.b) has NONE affinity, -** no conversion should be attempted before using a t2.b value as part of -** a key to search the index. Hence the first byte in the returned affinity -** string in this example would be set to SQLITE_AFF_NONE. +** Construct a trigger step that implements a DELETE statement and return +** a pointer to that trigger step. The parser calls this routine when it +** sees a DELETE statement inside the body of a CREATE TRIGGER. */ -static int codeAllEqualityTerms( - Parse *pParse, /* Parsing context */ - WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - WhereClause *pWC, /* The WHERE clause */ - Bitmask notReady, /* Which parts of FROM have not yet been coded */ - int nExtraReg, /* Number of extra registers to allocate */ - char **pzAff /* OUT: Set to point to affinity string */ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( + sqlite3 *db, /* Database connection */ + Token *pTableName, /* The table from which rows are deleted */ + Expr *pWhere /* The WHERE clause */ ){ - int nEq = pLevel->plan.nEq; /* The number of == or IN constraints to code */ - Vdbe *v = pParse->pVdbe; /* The vm under construction */ - Index *pIdx; /* The index being used for this loop */ - int iCur = pLevel->iTabCur; /* The cursor of the table */ - WhereTerm *pTerm; /* A single constraint term */ - int j; /* Loop counter */ - int regBase; /* Base register */ - int nReg; /* Number of registers to allocate */ - char *zAff; /* Affinity string to return */ - - /* This module is only called on query plans that use an index. */ - assert( pLevel->plan.wsFlags & WHERE_INDEXED ); - pIdx = pLevel->plan.u.pIdx; - - /* Figure out how many memory cells we will need then allocate them. - */ - regBase = pParse->nMem + 1; - nReg = pLevel->plan.nEq + nExtraReg; - pParse->nMem += nReg; + TriggerStep *pTriggerStep; - zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); - if( !zAff ){ - pParse->db->mallocFailed = 1; + pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName); + if( pTriggerStep ){ + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pTriggerStep->orconf = OE_Default; } + sqlite3ExprDelete(db, pWhere); + return pTriggerStep; +} - /* Evaluate the equality constraints - */ - assert( pIdx->nColumn>=nEq ); - for(j=0; jaiColumn[j]; - pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); - if( NEVER(pTerm==0) ) break; - /* The following true for indices with redundant columns. - ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ - testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); - if( r1!=regBase+j ){ - if( nReg==1 ){ - sqlite3ReleaseTempReg(pParse, regBase); - regBase = r1; - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); - } - } - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IN ); - if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ - Expr *pRight = pTerm->pExpr->pRight; - sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk); - if( zAff ){ - if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ - zAff[j] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ - zAff[j] = SQLITE_AFF_NONE; - } - } - } - } - *pzAff = zAff; - return regBase; +/* +** Recursively delete a Trigger structure +*/ +SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3 *db, Trigger *pTrigger){ + if( pTrigger==0 ) return; + sqlite3DeleteTriggerStep(db, pTrigger->step_list); + sqlite3DbFree(db, pTrigger->zName); + sqlite3DbFree(db, pTrigger->table); + sqlite3ExprDelete(db, pTrigger->pWhen); + sqlite3IdListDelete(db, pTrigger->pColumns); + sqlite3DbFree(db, pTrigger); } -#ifndef SQLITE_OMIT_EXPLAIN /* -** This routine is a helper for explainIndexRange() below +** This function is called to drop a trigger from the database schema. ** -** pStr holds the text of an expression that we are building up one term -** at a time. This routine adds a new term to the end of the expression. -** Terms are separated by AND so add the "AND" text for second and subsequent -** terms only. -*/ -static void explainAppendTerm( - StrAccum *pStr, /* The text expression being built */ - int iTerm, /* Index of this term. First is zero */ - const char *zColumn, /* Name of the column */ - const char *zOp /* Name of the operator */ -){ - if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3StrAccumAppend(pStr, zColumn, -1); - sqlite3StrAccumAppend(pStr, zOp, 1); - sqlite3StrAccumAppend(pStr, "?", 1); -} - -/* -** Argument pLevel describes a strategy for scanning table pTab. This -** function returns a pointer to a string buffer containing a description -** of the subset of table rows scanned by the strategy in the form of an -** SQL expression. Or, if all rows are scanned, NULL is returned. -** -** For example, if the query: -** -** SELECT * FROM t1 WHERE a=1 AND b>2; -** -** is run and there is an index on (a, b), then this function returns a -** string similar to: -** -** "a=? AND b>?" -** -** The returned pointer points to memory obtained from sqlite3DbMalloc(). -** It is the responsibility of the caller to free the buffer when it is -** no longer required. -*/ -static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ - WherePlan *pPlan = &pLevel->plan; - Index *pIndex = pPlan->u.pIdx; - int nEq = pPlan->nEq; - int i, j; - Column *aCol = pTab->aCol; - int *aiColumn = pIndex->aiColumn; - StrAccum txt; +** This may be called directly from the parser and therefore identifies +** the trigger by name. The sqlite3DropTriggerPtr() routine does the +** same job as this routine except it takes a pointer to the trigger +** instead of the trigger name. +**/ +SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ + Trigger *pTrigger = 0; + int i; + const char *zDb; + const char *zName; + int nName; + sqlite3 *db = pParse->db; - if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ - return 0; - } - sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); - txt.db = db; - sqlite3StrAccumAppend(&txt, " (", 2); - for(i=0; i"); + assert( pName->nSrc==1 ); + zDb = pName->a[0].zDatabase; + zName = pName->a[0].zName; + nName = sqlite3Strlen30(zName); + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); + if( pTrigger ) break; } - if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ - explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<"); + if( !pTrigger ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "no such trigger: %S", pName, 0); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, zDb); + } + pParse->checkSchema = 1; + goto drop_trigger_cleanup; } - sqlite3StrAccumAppend(&txt, ")", 1); - return sqlite3StrAccumFinish(&txt); + sqlite3DropTriggerPtr(pParse, pTrigger); + +drop_trigger_cleanup: + sqlite3SrcListDelete(db, pName); } /* -** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single -** record is added to the output to describe the table scan strategy in -** pLevel. +** Return a pointer to the Table structure for the table that a trigger +** is set on. */ -static void explainOneScan( - Parse *pParse, /* Parse context */ - SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ - u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ -){ - if( pParse->explain==2 ){ - u32 flags = pLevel->plan.wsFlags; - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - Vdbe *v = pParse->pVdbe; /* VM being constructed */ - sqlite3 *db = pParse->db; /* Database handle */ - char *zMsg; /* Text to add to EQP output */ - sqlite3_int64 nRow; /* Expected number of rows visited by scan */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ - int isSearch; /* True for a SEARCH. False for SCAN. */ +static Table *tableOfTrigger(Trigger *pTrigger){ + int n = sqlite3Strlen30(pTrigger->table); + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); +} - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; - isSearch = (pLevel->plan.nEq>0) - || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); +/* +** Drop a trigger given a pointer to that trigger. +*/ +SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ + Table *pTable; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; - zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); - if( pItem->pSelect ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); - }else{ - zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); + assert( iDb>=0 && iDbnDb ); + pTable = tableOfTrigger(pTrigger); + assert( pTable ); + assert( pTable->pSchema==pTrigger->pSchema || iDb==1 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_DROP_TRIGGER; + const char *zDb = db->aDb[iDb].zName; + const char *zTab = SCHEMA_TABLE(iDb); + if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; + if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || + sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + return; } + } +#endif - if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); - } - if( (flags & WHERE_INDEXED)!=0 ){ - char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); - zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, - ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), - ((flags & WHERE_IDX_ONLY)?"COVERING ":""), - ((flags & WHERE_TEMP_INDEX)?"":" "), - ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName), - zWhere - ); - sqlite3DbFree(db, zWhere); - }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + /* Generate code to destroy the database record of the trigger. + */ + assert( pTable!=0 ); + if( (v = sqlite3GetVdbe(pParse))!=0 ){ + int base; + static const VdbeOpList dropTrigger[] = { + { OP_Rewind, 0, ADDR(9), 0}, + { OP_String8, 0, 1, 0}, /* 1 */ + { OP_Column, 0, 1, 2}, + { OP_Ne, 2, ADDR(8), 1}, + { OP_String8, 0, 1, 0}, /* 4: "trigger" */ + { OP_Column, 0, 0, 2}, + { OP_Ne, 2, ADDR(8), 1}, + { OP_Delete, 0, 0, 0}, + { OP_Next, 0, ADDR(1), 0}, /* 8 */ + }; - if( flags&WHERE_ROWID_EQ ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); - }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); - }else if( flags&WHERE_TOP_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidplan.u.pVtabIdx; - zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, - pVtabIdx->idxNum, pVtabIdx->idxStr); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3OpenMasterTable(pParse, iDb); + base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger); + sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); + sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_Close, 0, 0); + sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); + if( pParse->nMem<3 ){ + pParse->nMem = 3; } -#endif - if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ - testcase( wctrlFlags & WHERE_ORDERBY_MIN ); - nRow = 1; - }else{ - nRow = (sqlite3_int64)pLevel->plan.nRow; + } +} + +/* +** Remove a trigger from the hash tables of the sqlite* pointer. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){ + Trigger *pTrigger; + Hash *pHash; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &(db->aDb[iDb].pSchema->trigHash); + pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); + if( ALWAYS(pTrigger) ){ + if( pTrigger->pSchema==pTrigger->pTabSchema ){ + Table *pTab = tableOfTrigger(pTrigger); + Trigger **pp; + for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); + *pp = (*pp)->pNext; } - zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); - sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + sqlite3DeleteTrigger(db, pTrigger); + db->flags |= SQLITE_InternChanges; } } -#else -# define explainOneScan(u,v,w,x,y,z) -#endif /* SQLITE_OMIT_EXPLAIN */ +/* +** pEList is the SET clause of an UPDATE statement. Each entry +** in pEList is of the format =. If any of the entries +** in pEList have an which matches an identifier in pIdList, +** then return TRUE. If pIdList==NULL, then it is considered a +** wildcard that matches anything. Likewise if pEList==NULL then +** it matches anything so always return true. Return false only +** if there is no match. +*/ +static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ + int e; + if( pIdList==0 || NEVER(pEList==0) ) return 1; + for(e=0; enExpr; e++){ + if( sqlite3IdListIndex(pIdList, pEList->a[e].zName)>=0 ) return 1; + } + return 0; +} /* -** Generate code for the start of the iLevel-th loop in the WHERE clause -** implementation described by pWInfo. +** Return a list of all triggers on table pTab if there exists at least +** one trigger that must be fired when an operation of type 'op' is +** performed on the table, and, if that operation is an UPDATE, if at +** least one of the columns in pChanges is being modified. */ -static Bitmask codeOneLoopStart( - WhereInfo *pWInfo, /* Complete information about the WHERE clause */ - int iLevel, /* Which level of pWInfo->a[] should be coded */ - u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ - Bitmask notReady /* Which tables are currently available */ +SQLITE_PRIVATE Trigger *sqlite3TriggersExist( + Parse *pParse, /* Parse context */ + Table *pTab, /* The table the contains the triggers */ + int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ + ExprList *pChanges, /* Columns that change in an UPDATE statement */ + int *pMask /* OUT: Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ ){ - int j, k; /* Loop counters */ - int iCur; /* The VDBE cursor for the table */ - int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ - int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ - WhereClause *pWC; /* Decomposition of the entire WHERE clause */ - WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ - Vdbe *v; /* The prepared stmt under constructions */ - struct SrcList_item *pTabItem; /* FROM clause term being coded */ - int addrBrk; /* Jump here to break out of the loop */ - int addrCont; /* Jump here to continue with next cycle */ - int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ - int iReleaseReg = 0; /* Temp register to free before returning */ + int mask = 0; + Trigger *pList = 0; + Trigger *p; - pParse = pWInfo->pParse; - v = pParse->pVdbe; - pWC = pWInfo->pWC; - pLevel = &pWInfo->a[iLevel]; - pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - iCur = pTabItem->iCursor; - bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; - omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 - && (wctrlFlags & WHERE_FORCE_TABLE)==0; + if( (pParse->db->flags & SQLITE_EnableTrigger)!=0 ){ + pList = sqlite3TriggerList(pParse, pTab); + } + assert( pList==0 || IsVirtual(pTab)==0 ); + for(p=pList; p; p=p->pNext){ + if( p->op==op && checkColumnOverlap(p->pColumns, pChanges) ){ + mask |= p->tr_tm; + } + } + if( pMask ){ + *pMask = mask; + } + return (mask ? pList : 0); +} - /* Create labels for the "break" and "continue" instructions - ** for the current loop. Jump to addrBrk to break out of a loop. - ** Jump to cont to go immediately to the next iteration of the - ** loop. - ** - ** When there is an IN operator, we also have a "addrNxt" label that - ** means to continue with the next IN value combination. When - ** there are no IN operators in the constraints, the "addrNxt" label - ** is the same as "addrBrk". - */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); +/* +** Convert the pStep->target token into a SrcList and return a pointer +** to that SrcList. +** +** This routine adds a specific database name, if needed, to the target when +** forming the SrcList. This prevents a trigger in one database from +** referring to a target in another database. An exception is when the +** trigger is in TEMP in which case it can refer to any other database it +** wants. +*/ +static SrcList *targetSrcList( + Parse *pParse, /* The parsing context */ + TriggerStep *pStep /* The trigger containing the target token */ +){ + int iDb; /* Index of the database to use */ + SrcList *pSrc; /* SrcList to be returned */ - /* If this is the right table of a LEFT OUTER JOIN, allocate and - ** initialize a memory cell that records if this table matches any - ** row of the left table of the join. - */ - if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ - pLevel->iLeftJoin = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + pSrc = sqlite3SrcListAppend(pParse->db, 0, &pStep->target, 0); + if( pSrc ){ + assert( pSrc->nSrc>0 ); + assert( pSrc->a!=0 ); + iDb = sqlite3SchemaToIndex(pParse->db, pStep->pTrig->pSchema); + if( iDb==0 || iDb>=2 ){ + sqlite3 *db = pParse->db; + assert( iDbdb->nDb ); + pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + } } + return pSrc; +} -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 0: The table is a virtual-table. Use the VFilter and VNext - ** to access the data. +/* +** Generate VDBE code for the statements inside the body of a single +** trigger. +*/ +static int codeTriggerProgram( + Parse *pParse, /* The parser context */ + TriggerStep *pStepList, /* List of statements inside the trigger body */ + int orconf /* Conflict algorithm. (OE_Abort, etc) */ +){ + TriggerStep *pStep; + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + + assert( pParse->pTriggerTab && pParse->pToplevel ); + assert( pStepList ); + assert( v!=0 ); + for(pStep=pStepList; pStep; pStep=pStep->pNext){ + /* Figure out the ON CONFLICT policy that will be used for this step + ** of the trigger program. If the statement that caused this trigger + ** to fire had an explicit ON CONFLICT, then use it. Otherwise, use + ** the ON CONFLICT policy that was specified as part of the trigger + ** step statement. Example: + ** + ** CREATE TRIGGER AFTER INSERT ON t1 BEGIN; + ** INSERT OR REPLACE INTO t2 VALUES(new.a, new.b); + ** END; + ** + ** INSERT INTO t1 ... ; -- insert into t2 uses REPLACE policy + ** INSERT OR IGNORE INTO t1 ... ; -- insert into t2 uses IGNORE policy */ - int iReg; /* P3 Value for OP_VFilter */ - sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; - int nConstraint = pVtabIdx->nConstraint; - struct sqlite3_index_constraint_usage *aUsage = - pVtabIdx->aConstraintUsage; - const struct sqlite3_index_constraint *aConstraint = - pVtabIdx->aConstraint; + pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; - sqlite3ExprCachePush(pParse); - iReg = sqlite3GetTempRange(pParse, nConstraint+2); - for(j=1; j<=nConstraint; j++){ - for(k=0; ka[iTerm].pExpr->pRight, iReg+j+1); - break; - } + switch( pStep->op ){ + case TK_UPDATE: { + sqlite3Update(pParse, + targetSrcList(pParse, pStep), + sqlite3ExprListDup(db, pStep->pExprList, 0), + sqlite3ExprDup(db, pStep->pWhere, 0), + pParse->eOrconf + ); + break; } - if( k==nConstraint ) break; - } - sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, - pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); - pVtabIdx->needToFreeIdxStr = 0; - for(j=0; ja[iTerm]); + case TK_INSERT: { + sqlite3Insert(pParse, + targetSrcList(pParse, pStep), + sqlite3ExprListDup(db, pStep->pExprList, 0), + sqlite3SelectDup(db, pStep->pSelect, 0), + sqlite3IdListDup(db, pStep->pIdList), + pParse->eOrconf + ); + break; + } + case TK_DELETE: { + sqlite3DeleteFrom(pParse, + targetSrcList(pParse, pStep), + sqlite3ExprDup(db, pStep->pWhere, 0) + ); + break; + } + default: assert( pStep->op==TK_SELECT ); { + SelectDest sDest; + Select *pSelect = sqlite3SelectDup(db, pStep->pSelect, 0); + sqlite3SelectDestInit(&sDest, SRT_Discard, 0); + sqlite3Select(pParse, pSelect, &sDest); + sqlite3SelectDelete(db, pSelect); + break; } + } + if( pStep->op!=TK_SELECT ){ + sqlite3VdbeAddOp0(v, OP_ResetCount); } - pLevel->op = OP_VNext; - pLevel->p1 = iCur; - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse, 1); - }else -#endif /* SQLITE_OMIT_VIRTUALTABLE */ + } - if( pLevel->plan.wsFlags & WHERE_ROWID_EQ ){ - /* Case 1: We can directly reference a single row using an - ** equality comparison against the ROWID field. Or - ** we reference multiple rows using a "rowid IN (...)" - ** construct. - */ - iReleaseReg = sqlite3GetTempReg(pParse); - pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); - assert( pTerm!=0 ); - assert( pTerm->pExpr!=0 ); - assert( pTerm->leftCursor==iCur ); - assert( omitTable==0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); - addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); - pLevel->op = OP_Noop; - }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){ - /* Case 2: We have an inequality comparison against the ROWID field. - */ - int testOp = OP_Noop; - int start; - int memEndValue = 0; - WhereTerm *pStart, *pEnd; + return 0; +} - assert( omitTable==0 ); - pStart = findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0); - pEnd = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0); - if( bRev ){ - pTerm = pStart; - pStart = pEnd; - pEnd = pTerm; - } - if( pStart ){ - Expr *pX; /* The expression that defines the start bound */ - int r1, rTemp; /* Registers for holding the start boundary */ +#ifdef SQLITE_DEBUG +/* +** This function is used to add VdbeComment() annotations to a VDBE +** program. It is not used in production code, only for debugging. +*/ +static const char *onErrorText(int onError){ + switch( onError ){ + case OE_Abort: return "abort"; + case OE_Rollback: return "rollback"; + case OE_Fail: return "fail"; + case OE_Replace: return "replace"; + case OE_Ignore: return "ignore"; + case OE_Default: return "default"; + } + return "n/a"; +} +#endif - /* The following constant maps TK_xx codes into corresponding - ** seek opcodes. It depends on a particular ordering of TK_xx - */ - const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGt, - /* TK_LE */ OP_SeekLe, - /* TK_LT */ OP_SeekLt, - /* TK_GE */ OP_SeekGe - }; - assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ - assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ +/* +** Parse context structure pFrom has just been used to create a sub-vdbe +** (trigger program). If an error has occurred, transfer error information +** from pFrom to pTo. +*/ +static void transferParseError(Parse *pTo, Parse *pFrom){ + assert( pFrom->zErrMsg==0 || pFrom->nErr ); + assert( pTo->zErrMsg==0 || pTo->nErr ); + if( pTo->nErr==0 ){ + pTo->zErrMsg = pFrom->zErrMsg; + pTo->nErr = pFrom->nErr; + }else{ + sqlite3DbFree(pFrom->db, pFrom->zErrMsg); + } +} - testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - pX = pStart->pExpr; - assert( pX!=0 ); - assert( pStart->leftCursor==iCur ); - r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); - sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); - VdbeComment((v, "pk")); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); - sqlite3ReleaseTempReg(pParse, rTemp); - disableTerm(pLevel, pStart); - }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); - } - if( pEnd ){ - Expr *pX; - pX = pEnd->pExpr; - assert( pX!=0 ); - assert( pEnd->leftCursor==iCur ); - testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - memEndValue = ++pParse->nMem; - sqlite3ExprCode(pParse, pX->pRight, memEndValue); - if( pX->op==TK_LT || pX->op==TK_GT ){ - testOp = bRev ? OP_Le : OP_Ge; - }else{ - testOp = bRev ? OP_Lt : OP_Gt; - } - disableTerm(pLevel, pEnd); - } - start = sqlite3VdbeCurrentAddr(v); - pLevel->op = bRev ? OP_Prev : OP_Next; - pLevel->p1 = iCur; - pLevel->p2 = start; - if( pStart==0 && pEnd==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); - } - if( testOp!=OP_Noop ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); - sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); - } - }else if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){ - /* Case 3: A scan using an index. - ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: - ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 - ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: - ** - ** x=5 AND z<10 - ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. - ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. - */ - static const u8 aStartOp[] = { - 0, - 0, - OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ - OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGt, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLt, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ - }; - static const u8 aEndOp[] = { - OP_Noop, /* 0: (!end_constraints) */ - OP_IdxGE, /* 1: (end_constraints && !bRev) */ - OP_IdxLT /* 2: (end_constraints && bRev) */ - }; - int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ - int regBase; /* Base register holding constraint values */ - int r1; /* Temp register */ - WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ - WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ - int startEq; /* True if range start uses ==, >= or <= */ - int endEq; /* True if range end uses ==, >= or <= */ - int start_constraints; /* Start of range is constrained */ - int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ - char *zStartAff; /* Affinity for start of range constraint */ - char *zEndAff; /* Affinity for end of range constraint */ +/* +** Create and populate a new TriggerPrg object with a sub-program +** implementing trigger pTrigger with ON CONFLICT policy orconf. +*/ +static TriggerPrg *codeRowTrigger( + Parse *pParse, /* Current parse context */ + Trigger *pTrigger, /* Trigger to code */ + Table *pTab, /* The table pTrigger is attached to */ + int orconf /* ON CONFLICT policy to code trigger program with */ +){ + Parse *pTop = sqlite3ParseToplevel(pParse); + sqlite3 *db = pParse->db; /* Database handle */ + TriggerPrg *pPrg; /* Value to return */ + Expr *pWhen = 0; /* Duplicate of trigger WHEN expression */ + Vdbe *v; /* Temporary VM */ + NameContext sNC; /* Name context for sub-vdbe */ + SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ + Parse *pSubParse; /* Parse context for sub-vdbe */ + int iEndTrigger = 0; /* Label to jump to if WHEN is false */ - pIdx = pLevel->plan.u.pIdx; - iIdxCur = pLevel->iIdxCur; - k = pIdx->aiColumn[nEq]; /* Column for inequality constraints */ + assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + assert( pTop->pVdbe ); - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERBY) - && (pIdx->nColumn>nEq) - ){ - /* assert( pOrderBy->nExpr==1 ); */ - /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */ - isMinQuery = 1; - nExtraReg = 1; - } + /* Allocate the TriggerPrg and SubProgram objects. To ensure that they + ** are freed if an error occurs, link them into the Parse.pTriggerPrg + ** list of the top-level Parse object sooner rather than later. */ + pPrg = sqlite3DbMallocZero(db, sizeof(TriggerPrg)); + if( !pPrg ) return 0; + pPrg->pNext = pTop->pTriggerPrg; + pTop->pTriggerPrg = pPrg; + pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); + if( !pProgram ) return 0; + sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); + pPrg->pTrigger = pTrigger; + pPrg->orconf = orconf; + pPrg->aColmask[0] = 0xffffffff; + pPrg->aColmask[1] = 0xffffffff; - /* Find any inequality constraint terms for the start and end - ** of the range. - */ - if( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = findTerm(pWC, iCur, k, notReady, (WO_LT|WO_LE), pIdx); - nExtraReg = 1; - } - if( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = findTerm(pWC, iCur, k, notReady, (WO_GT|WO_GE), pIdx); - nExtraReg = 1; - } + /* Allocate and populate a new Parse context to use for coding the + ** trigger sub-program. */ + pSubParse = sqlite3StackAllocZero(db, sizeof(Parse)); + if( !pSubParse ) return 0; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pSubParse; + pSubParse->db = db; + pSubParse->pTriggerTab = pTab; + pSubParse->pToplevel = pTop; + pSubParse->zAuthContext = pTrigger->zName; + pSubParse->eTriggerOp = pTrigger->op; + pSubParse->nQueryLoop = pParse->nQueryLoop; - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. - */ - regBase = codeAllEqualityTerms( - pParse, pLevel, pWC, notReady, nExtraReg, &zStartAff + v = sqlite3GetVdbe(pSubParse); + if( v ){ + VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", + pTrigger->zName, onErrorText(orconf), + (pTrigger->tr_tm==TRIGGER_BEFORE ? "BEFORE" : "AFTER"), + (pTrigger->op==TK_UPDATE ? "UPDATE" : ""), + (pTrigger->op==TK_INSERT ? "INSERT" : ""), + (pTrigger->op==TK_DELETE ? "DELETE" : ""), + pTab->zName + )); +#ifndef SQLITE_OMIT_TRACE + sqlite3VdbeChangeP4(v, -1, + sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC ); - zEndAff = sqlite3DbStrDup(pParse->db, zStartAff); - addrNxt = pLevel->addrNxt; +#endif - /* If we are doing a reverse order scan on an ascending index, or - ** a forward order scan on a descending index, interchange the - ** start and end terms (pRangeStart and pRangeEnd). - */ - if( nEqnColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC) ){ - SWAP(WhereTerm *, pRangeEnd, pRangeStart); + /* If one was specified, code the WHEN clause. If it evaluates to false + ** (or NULL) the sub-vdbe is immediately halted by jumping to the + ** OP_Halt inserted at the end of the program. */ + if( pTrigger->pWhen ){ + pWhen = sqlite3ExprDup(db, pTrigger->pWhen, 0); + if( SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) + && db->mallocFailed==0 + ){ + iEndTrigger = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pWhen); } - testcase( pRangeStart && pRangeStart->eOperator & WO_LE ); - testcase( pRangeStart && pRangeStart->eOperator & WO_GE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_LE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_GE ); - startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); - endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); - start_constraints = pRangeStart || nEq>0; + /* Code the trigger program into the sub-vdbe. */ + codeTriggerProgram(pSubParse, pTrigger->step_list, orconf); - /* Seek the index cursor to the start of the range. */ - nConstraint = nEq; - if( pRangeStart ){ - Expr *pRight = pRangeStart->pExpr->pRight; - sqlite3ExprCode(pParse, pRight, regBase+nEq); - sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); - if( zStartAff ){ - if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - } - nConstraint++; - testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - }else if( isMinQuery ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - nConstraint++; - startEq = 0; - start_constraints = 1; + /* Insert an OP_Halt at the end of the sub-program. */ + if( iEndTrigger ){ + sqlite3VdbeResolveLabel(v, iEndTrigger); } - codeApplyAffinity(pParse, regBase, nConstraint, zStartAff); - op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; - assert( op!=0 ); - testcase( op==OP_Rewind ); - testcase( op==OP_Last ); - testcase( op==OP_SeekGt ); - testcase( op==OP_SeekGe ); - testcase( op==OP_SeekLe ); - testcase( op==OP_SeekLt ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + sqlite3VdbeAddOp0(v, OP_Halt); + VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); - /* Load the value for the inequality constraint at the end of the - ** range (if any). - */ - nConstraint = nEq; - if( pRangeEnd ){ - Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - sqlite3ExprCode(pParse, pRight, regBase+nEq); - sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); - if( zEndAff ){ - if( sqlite3CompareAffinity(pRight, zEndAff[nEq])==SQLITE_AFF_NONE){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zEndAff[nEq] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zEndAff[nEq]) ){ - zEndAff[nEq] = SQLITE_AFF_NONE; - } - } - codeApplyAffinity(pParse, regBase, nEq+1, zEndAff); - nConstraint++; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + transferParseError(pParse, pSubParse); + if( db->mallocFailed==0 ){ + pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); } - sqlite3DbFree(pParse->db, zStartAff); - sqlite3DbFree(pParse->db, zEndAff); + pProgram->nMem = pSubParse->nMem; + pProgram->nCsr = pSubParse->nTab; + pProgram->token = (void *)pTrigger; + pPrg->aColmask[0] = pSubParse->oldmask; + pPrg->aColmask[1] = pSubParse->newmask; + sqlite3VdbeDelete(v); + } - /* Top of the loop body */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); + assert( !pSubParse->pAinc && !pSubParse->pZombieTab ); + assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); + sqlite3StackFree(db, pSubParse); - /* Check if the index cursor is past the end of the range. */ - op = aEndOp[(pRangeEnd || nEq) * (1 + bRev)]; - testcase( op==OP_Noop ); - testcase( op==OP_IdxGE ); - testcase( op==OP_IdxLT ); - if( op!=OP_Noop ){ - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0); - } + return pPrg; +} + +/* +** Return a pointer to a TriggerPrg object containing the sub-program for +** trigger pTrigger with default ON CONFLICT algorithm orconf. If no such +** TriggerPrg object exists, a new object is allocated and populated before +** being returned. +*/ +static TriggerPrg *getRowTrigger( + Parse *pParse, /* Current parse context */ + Trigger *pTrigger, /* Trigger to code */ + Table *pTab, /* The table trigger pTrigger is attached to */ + int orconf /* ON CONFLICT algorithm. */ +){ + Parse *pRoot = sqlite3ParseToplevel(pParse); + TriggerPrg *pPrg; - /* If there are inequality constraints, check that the value - ** of the table column that the inequality contrains is not NULL. - ** If it is, jump to the next iteration of the loop. - */ - r1 = sqlite3GetTempReg(pParse); - testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); - testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); - if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); - sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); - } - sqlite3ReleaseTempReg(pParse, r1); + assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); - /* Seek the table cursor, if required */ - disableTerm(pLevel, pRangeStart); - disableTerm(pLevel, pRangeEnd); - if( !omitTable ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ - } + /* It may be that this trigger has already been coded (or is in the + ** process of being coded). If this is the case, then an entry with + ** a matching TriggerPrg.pTrigger field will be present somewhere + ** in the Parse.pTriggerPrg list. Search for such an entry. */ + for(pPrg=pRoot->pTriggerPrg; + pPrg && (pPrg->pTrigger!=pTrigger || pPrg->orconf!=orconf); + pPrg=pPrg->pNext + ); - /* Record the instruction used to terminate the loop. Disable - ** WHERE clause terms made redundant by the index range scan. - */ - pLevel->op = bRev ? OP_Prev : OP_Next; - pLevel->p1 = iIdxCur; - }else + /* If an existing TriggerPrg could not be located, create a new one. */ + if( !pPrg ){ + pPrg = codeRowTrigger(pParse, pTrigger, pTab, orconf); + } -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ - /* Case 4: Two or more separately indexed terms connected by OR - ** - ** Example: - ** - ** CREATE TABLE t1(a,b,c,d); - ** CREATE INDEX i1 ON t1(a); - ** CREATE INDEX i2 ON t1(b); - ** CREATE INDEX i3 ON t1(c); - ** - ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) - ** - ** In the example, there are three indexed terms connected by OR. - ** The top of the loop looks like this: - ** - ** Null 1 # Zero the rowset in reg 1 - ** - ** Then, for each indexed term, the following. The arguments to - ** RowSetTest are such that the rowid of the current row is inserted - ** into the RowSet. If it is already present, control skips the - ** Gosub opcode and jumps straight to the code generated by WhereEnd(). - ** - ** sqlite3WhereBegin() - ** RowSetTest # Insert rowid into rowset - ** Gosub 2 A - ** sqlite3WhereEnd() - ** - ** Following the above, code to terminate the loop. Label A, the target - ** of the Gosub above, jumps to the instruction right after the Goto. - ** - ** Null 1 # Zero the rowset in reg 1 - ** Goto B # The loop is finished. - ** - ** A: # Return data, whatever. - ** - ** Return 2 # Jump back to the Gosub - ** - ** B: - ** - */ - WhereClause *pOrWc; /* The OR-clause broken out into subterms */ - WhereTerm *pFinal; /* Final subterm within the OR-clause. */ - SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + return pPrg; +} - int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ - int regRowset = 0; /* Register for RowSet object */ - int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ - int iRetInit; /* Address of regReturn init */ - int untestedTerms = 0; /* Some terms not completely tested */ - int ii; - - pTerm = pLevel->plan.u.pTerm; - assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); - assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); - pOrWc = &pTerm->u.pOrInfo->wc; - pFinal = &pOrWc->a[pOrWc->nTerm-1]; - pLevel->op = OP_Return; - pLevel->p1 = regReturn; +/* +** Generate code for the trigger program associated with trigger p on +** table pTab. The reg, orconf and ignoreJump parameters passed to this +** function are the same as those described in the header function for +** sqlite3CodeRowTrigger() +*/ +SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( + Parse *pParse, /* Parse context */ + Trigger *p, /* Trigger to code */ + Table *pTab, /* The table to code triggers from */ + int reg, /* Reg array containing OLD.* and NEW.* values */ + int orconf, /* ON CONFLICT policy */ + int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); /* Main VM */ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); - /* Set up a new SrcList ni pOrTab containing the table being scanned - ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. - ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). - */ - if( pWInfo->nLevel>1 ){ - int nNotReady; /* The number of notReady tables */ - struct SrcList_item *origSrc; /* Original list of tables */ - nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(pParse->db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); - if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (i16)(nNotReady + 1); - pOrTab->nSrc = pOrTab->nAlloc; - memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); - origSrc = pWInfo->pTabList->a; - for(k=1; k<=nNotReady; k++){ - memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); - } - }else{ - pOrTab = pWInfo->pTabList; - } + /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program + ** is a pointer to the sub-vdbe containing the trigger program. */ + if( pPrg ){ + int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); - /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. - ** - ** Also initialize regReturn to contain the address of the instruction - ** immediately following the OP_Return at the bottom of the loop. This - ** is required in a few obscure LEFT JOIN cases where control jumps - ** over the top of the loop into the body of it. In this case the - ** correct response for the end-of-loop code (the OP_Return) is to - ** fall through to the next instruction, just as an OP_Next does if - ** called on an uninitialized cursor. - */ - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - regRowset = ++pParse->nMem; - regRowid = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); - } - iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); - - for(ii=0; iinTerm; ii++){ - WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ - /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, - WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); - if( pSubWInfo ){ - explainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); - int r; - r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); - sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, - sqlite3VdbeCurrentAddr(v)+2, r, iSet); - } - sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); + sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); + VdbeComment( + (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); - /* The pSubWInfo->untestedTerms flag means that this OR term - ** contained one or more AND term from a notReady table. The - ** terms from the notReady table could not be tested and will - ** need to be tested later. - */ - if( pSubWInfo->untestedTerms ) untestedTerms = 1; + /* Set the P5 operand of the OP_Program instruction to non-zero if + ** recursive invocation of this trigger program is disallowed. Recursive + ** invocation is disallowed if (a) the sub-program is really a trigger, + ** not a foreign key action, and (b) the flag to enable recursive triggers + ** is clear. */ + sqlite3VdbeChangeP5(v, (u8)bRecursive); + } +} - /* Finish the loop through table entries that match term pOrTerm. */ - sqlite3WhereEnd(pSubWInfo); - } - } - } - sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); - sqlite3VdbeResolveLabel(v, iLoopBody); +/* +** This is called to code the required FOR EACH ROW triggers for an operation +** on table pTab. The operation to code triggers for (INSERT, UPDATE or DELETE) +** is given by the op paramater. The tr_tm parameter determines whether the +** BEFORE or AFTER triggers are coded. If the operation is an UPDATE, then +** parameter pChanges is passed the list of columns being modified. +** +** If there are no triggers that fire at the specified time for the specified +** operation on pTab, this function is a no-op. +** +** The reg argument is the address of the first in an array of registers +** that contain the values substituted for the new.* and old.* references +** in the trigger program. If N is the number of columns in table pTab +** (a copy of pTab->nCol), then registers are populated as follows: +** +** Register Contains +** ------------------------------------------------------ +** reg+0 OLD.rowid +** reg+1 OLD.* value of left-most column of pTab +** ... ... +** reg+N OLD.* value of right-most column of pTab +** reg+N+1 NEW.rowid +** reg+N+2 OLD.* value of left-most column of pTab +** ... ... +** reg+N+N+1 NEW.* value of right-most column of pTab +** +** For ON DELETE triggers, the registers containing the NEW.* values will +** never be accessed by the trigger program, so they are not allocated or +** populated by the caller (there is no data to populate them with anyway). +** Similarly, for ON INSERT triggers the values stored in the OLD.* registers +** are never accessed, and so are not allocated by the caller. So, for an +** ON INSERT trigger, the value passed to this function as parameter reg +** is not a readable register, although registers (reg+N) through +** (reg+N+N+1) are. +** +** Parameter orconf is the default conflict resolution algorithm for the +** trigger program to use (REPLACE, IGNORE etc.). Parameter ignoreJump +** is the instruction that control should jump to if a trigger program +** raises an IGNORE exception. +*/ +SQLITE_PRIVATE void sqlite3CodeRowTrigger( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* List of triggers on table pTab */ + int op, /* One of TK_UPDATE, TK_INSERT, TK_DELETE */ + ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int tr_tm, /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ + Table *pTab, /* The table to code triggers from */ + int reg, /* The first in an array of registers (see above) */ + int orconf, /* ON CONFLICT policy */ + int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +){ + Trigger *p; /* Used to iterate through pTrigger list */ - if( pWInfo->nLevel>1 ) sqlite3StackFree(pParse->db, pOrTab); - if( !untestedTerms ) disableTerm(pLevel, pTerm); - }else -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + assert( op==TK_UPDATE || op==TK_INSERT || op==TK_DELETE ); + assert( tr_tm==TRIGGER_BEFORE || tr_tm==TRIGGER_AFTER ); + assert( (op==TK_UPDATE)==(pChanges!=0) ); - { - /* Case 5: There is no usable index. We must do a complete - ** scan of the entire table. - */ - static const u8 aStep[] = { OP_Next, OP_Prev }; - static const u8 aStart[] = { OP_Rewind, OP_Last }; - assert( bRev==0 || bRev==1 ); - assert( omitTable==0 ); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - } - notReady &= ~getMask(pWC->pMaskSet, iCur); + for(p=pTrigger; p; p=p->pNext){ - /* Insert code to test every subexpression that can be completely - ** computed using the current set of tables. - ** - ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through - ** the use of indices become tests that are evaluated against each row of - ** the relevant input tables. - */ - k = 0; - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - k = 1; - pTerm->wtFlags |= TERM_CODED; - } + /* Sanity checking: The schema for the trigger and for the table are + ** always defined. The trigger must be in the same schema as the table + ** or else it must be a TEMP trigger. */ + assert( p->pSchema!=0 ); + assert( p->pTabSchema!=0 ); + assert( p->pSchema==p->pTabSchema + || p->pSchema==pParse->db->aDb[1].pSchema ); - /* For a LEFT OUTER JOIN, generate code that will record the fact that - ** at least one row of the right table has matched the left table. - */ - if( pLevel->iLeftJoin ){ - pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); - VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ - assert( pWInfo->untestedTerms ); - continue; - } - assert( pTerm->pExpr ); - sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; + /* Determine whether we should code this trigger */ + if( p->op==op + && p->tr_tm==tr_tm + && checkColumnOverlap(p->pColumns, pChanges) + ){ + sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); } } - sqlite3ReleaseTempReg(pParse, iReleaseReg); - - return notReady; } -#if defined(SQLITE_TEST) /* -** The following variable holds a text description of query plan generated -** by the most recent call to sqlite3WhereBegin(). Each call to WhereBegin -** overwrites the previous. This information is used for testing and -** analysis only. +** Triggers may access values stored in the old.* or new.* pseudo-table. +** This function returns a 32-bit bitmask indicating which columns of the +** old.* or new.* tables actually are used by triggers. This information +** may be used by the caller, for example, to avoid having to load the entire +** old.* record into memory when executing an UPDATE or DELETE command. +** +** Bit 0 of the returned mask is set if the left-most column of the +** table may be accessed using an [old|new].reference. Bit 1 is set if +** the second leftmost column value is required, and so on. If there +** are more than 32 columns in the table, and at least one of the columns +** with an index greater than 32 may be accessed, 0xffffffff is returned. +** +** It is not possible to determine if the old.rowid or new.rowid column is +** accessed by triggers. The caller must always assume that it is. +** +** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned +** applies to the old.* table. If 1, the new.* table. +** +** Parameter tr_tm must be a mask with one or both of the TRIGGER_BEFORE +** and TRIGGER_AFTER bits set. Values accessed by BEFORE triggers are only +** included in the returned mask if the TRIGGER_BEFORE bit is set in the +** tr_tm parameter. Similarly, values accessed by AFTER triggers are only +** included in the returned mask if the TRIGGER_AFTER bit is set in tr_tm. */ -SQLITE_API char sqlite3_query_plan[BMS*2*40]; /* Text of the join */ -static int nQPlan = 0; /* Next free slow in _query_plan[] */ - -#endif /* SQLITE_TEST */ - +SQLITE_PRIVATE u32 sqlite3TriggerColmask( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* List of triggers on table pTab */ + ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int isNew, /* 1 for new.* ref mask, 0 for old.* ref mask */ + int tr_tm, /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ + Table *pTab, /* The table to code triggers from */ + int orconf /* Default ON CONFLICT policy for trigger steps */ +){ + const int op = pChanges ? TK_UPDATE : TK_DELETE; + u32 mask = 0; + Trigger *p; -/* -** Free a WhereInfo structure -*/ -static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ - if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - sqlite3_index_info *pInfo = pWInfo->a[i].pIdxInfo; - if( pInfo ){ - /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */ - if( pInfo->needToFreeIdxStr ){ - sqlite3_free(pInfo->idxStr); - } - sqlite3DbFree(db, pInfo); - } - if( pWInfo->a[i].plan.wsFlags & WHERE_TEMP_INDEX ){ - Index *pIdx = pWInfo->a[i].plan.u.pIdx; - if( pIdx ){ - sqlite3DbFree(db, pIdx->zColAff); - sqlite3DbFree(db, pIdx); - } + assert( isNew==1 || isNew==0 ); + for(p=pTrigger; p; p=p->pNext){ + if( p->op==op && (tr_tm&p->tr_tm) + && checkColumnOverlap(p->pColumns,pChanges) + ){ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + if( pPrg ){ + mask |= pPrg->aColmask[isNew]; } } - whereClauseClear(pWInfo->pWC); - sqlite3DbFree(db, pWInfo); } + + return mask; } +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +/************** End of trigger.c *********************************************/ +/************** Begin file update.c ******************************************/ /* -** Generate the beginning of the loop used for WHERE clause processing. -** The return value is a pointer to an opaque structure that contains -** information needed to terminate the loop. Later, the calling routine -** should invoke sqlite3WhereEnd() with the return value of this function -** in order to complete the WHERE clause processing. -** -** If an error occurs, this routine returns NULL. -** -** The basic idea is to do a nested loop, one loop for each table in -** the FROM clause of a select. (INSERT and UPDATE statements are the -** same as a SELECT with only a single table in the FROM clause.) For -** example, if the SQL is this: -** -** SELECT * FROM t1, t2, t3 WHERE ...; -** -** Then the code generated is conceptually like the following: -** -** foreach row1 in t1 do \ Code generated -** foreach row2 in t2 do |-- by sqlite3WhereBegin() -** foreach row3 in t3 do / -** ... -** end \ Code generated -** end |-- by sqlite3WhereEnd() -** end / -** -** Note that the loops might not be nested in the order in which they -** appear in the FROM clause if a different order is better able to make -** use of indices. Note also that when the IN operator appears in -** the WHERE clause, it might result in additional nested loops for -** scanning through all values on the right-hand side of the IN. -** -** There are Btree cursors associated with each table. t1 uses cursor -** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. -** And so forth. This routine generates code to open those VDBE cursors -** and sqlite3WhereEnd() generates the code to close them. -** -** The code that sqlite3WhereBegin() generates leaves the cursors named -** in pTabList pointing at their appropriate entries. The [...] code -** can use OP_Column and OP_Rowid opcodes on these cursors to extract -** data from the various tables of the loop. -** -** If the WHERE clause is empty, the foreach loops must each scan their -** entire tables. Thus a three-way join is an O(N^3) operation. But if -** the tables have indices and there are terms in the WHERE clause that -** refer to those indices, a complete table scan can be avoided and the -** code will run much faster. Most of the work of this routine is checking -** to see if there are indices that can be used to speed up the loop. +** 2001 September 15 ** -** Terms of the WHERE clause are also used to limit which rows actually -** make it to the "..." in the middle of the loop. After each "foreach", -** terms of the WHERE clause that use only terms in that loop and outer -** loops are evaluated and if false a jump is made around all subsequent -** inner loops (or around the "..." if the test occurs within the inner- -** most loop) +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** OUTER JOINS +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** An outer join of tables t1 and t2 is conceptally coded as follows: +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle UPDATE statements. +*/ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Forward declaration */ +static void updateVirtualTable( + Parse *pParse, /* The parsing context */ + SrcList *pSrc, /* The virtual table to be modified */ + Table *pTab, /* The virtual table */ + ExprList *pChanges, /* The columns to change in the UPDATE statement */ + Expr *pRowidExpr, /* Expression used to recompute the rowid */ + int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ +); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** The most recently coded instruction was an OP_Column to retrieve the +** i-th column of table pTab. This routine sets the P4 parameter of the +** OP_Column to the default value, if any. ** -** foreach row1 in t1 do -** flag = 0 -** foreach row2 in t2 do -** start: -** ... -** flag = 1 -** end -** if flag==0 then -** move the row2 cursor to a null row -** goto start -** fi -** end +** The default value of a column is specified by a DEFAULT clause in the +** column definition. This was either supplied by the user when the table +** was created, or added later to the table definition by an ALTER TABLE +** command. If the latter, then the row-records in the table btree on disk +** may not contain a value for the column and the default value, taken +** from the P4 parameter of the OP_Column instruction, is returned instead. +** If the former, then all row-records are guaranteed to include a value +** for the column and the P4 value is not required. ** -** ORDER BY CLAUSE PROCESSING +** Column definitions created by an ALTER TABLE command may only have +** literal default values specified: a number, null or a string. (If a more +** complicated default expression value was provided, it is evaluated +** when the ALTER TABLE is executed and one of the literal values written +** into the sqlite_master table.) ** -** *ppOrderBy is a pointer to the ORDER BY clause of a SELECT statement, -** if there is one. If there is no ORDER BY clause or if this routine -** is called from an UPDATE or DELETE statement, then ppOrderBy is NULL. +** Therefore, the P4 parameter is only required if the default value for +** the column is a literal number, string or null. The sqlite3ValueFromExpr() +** function is capable of transforming these types of expressions into +** sqlite3_value objects. ** -** If an index can be used so that the natural output order of the table -** scan is correct for the ORDER BY clause, then that index is used and -** *ppOrderBy is set to NULL. This is an optimization that prevents an -** unnecessary sort of the result set if an index appropriate for the -** ORDER BY clause already exists. +** If parameter iReg is not negative, code an OP_RealAffinity instruction +** on register iReg. This is used when an equivalent integer value is +** stored in place of an 8-byte floating point value in order to save +** space. +*/ +SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + assert( pTab!=0 ); + if( !pTab->pSelect ){ + sqlite3_value *pValue; + u8 enc = ENC(sqlite3VdbeDb(v)); + Column *pCol = &pTab->aCol[i]; + VdbeComment((v, "%s.%s", pTab->zName, pCol->zName)); + assert( inCol ); + sqlite3ValueFromExpr(sqlite3VdbeDb(v), pCol->pDflt, enc, + pCol->affinity, &pValue); + if( pValue ){ + sqlite3VdbeChangeP4(v, -1, (const char *)pValue, P4_MEM); + } +#ifndef SQLITE_OMIT_FLOATING_POINT + if( iReg>=0 && pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } +#endif + } +} + +/* +** Process an UPDATE statement. ** -** If the where clause loops cannot be arranged to provide the correct -** output order, then the *ppOrderBy is unchanged. +** UPDATE OR IGNORE table_wxyz SET a=b, c=d WHERE e<5 AND f NOT NULL; +** \_______/ \________/ \______/ \________________/ +* onError pTabList pChanges pWhere */ -SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( - Parse *pParse, /* The parser context */ - SrcList *pTabList, /* A list of all tables to be scanned */ - Expr *pWhere, /* The WHERE clause */ - ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ - u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ +SQLITE_PRIVATE void sqlite3Update( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table in which we should change things */ + ExprList *pChanges, /* Things to be changed */ + Expr *pWhere, /* The WHERE clause. May be null */ + int onError /* How to handle constraint errors */ ){ - int i; /* Loop counter */ - int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ - int nTabList; /* Number of elements in pTabList */ - WhereInfo *pWInfo; /* Will become the return value of this function */ - Vdbe *v = pParse->pVdbe; /* The virtual database engine */ - Bitmask notReady; /* Cursors that are not yet positioned */ - WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereClause *pWC; /* Decomposition of the WHERE clause */ - struct SrcList_item *pTabItem; /* A single entry from pTabList */ - WhereLevel *pLevel; /* A single level in the pWInfo list */ - int iFrom; /* First unused FROM clause element */ - int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ - sqlite3 *db; /* Database connection */ + int i, j; /* Loop counters */ + Table *pTab; /* The table to be updated */ + int addr = 0; /* VDBE instruction address of the start of the loop */ + WhereInfo *pWInfo; /* Information about the WHERE clause */ + Vdbe *v; /* The virtual database engine */ + Index *pIdx; /* For looping over indices */ + int nIdx; /* Number of indices that need updating */ + int iCur; /* VDBE Cursor number of pTab */ + sqlite3 *db; /* The database structure */ + int *aRegIdx = 0; /* One register assigned to each index to be updated */ + int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the + ** an expression for the i-th column of the table. + ** aXRef[i]==-1 if the i-th column is not changed. */ + int chngRowid; /* True if the record number is being changed */ + Expr *pRowidExpr = 0; /* Expression defining the new record number */ + int openAll = 0; /* True if all indices need to be opened */ + AuthContext sContext; /* The authorization context */ + NameContext sNC; /* The name-context to resolve expressions in */ + int iDb; /* Database containing the table being updated */ + int okOnePass; /* True for one-pass algorithm without the FIFO */ + int hasFK; /* True if foreign key processing is required */ - /* The number of tables in the FROM clause is limited by the number of - ** bits in a Bitmask - */ - testcase( pTabList->nSrc==BMS ); - if( pTabList->nSrc>BMS ){ - sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); - return 0; +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True when updating a view (INSTEAD OF trigger) */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +#endif + int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */ + + /* Register Allocations */ + int regRowCount = 0; /* A count of rows changed */ + int regOldRowid; /* The old rowid */ + int regNewRowid; /* The new rowid */ + int regNew; + int regOld = 0; + int regRowSet = 0; /* Rowset of rows to be updated */ + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto update_cleanup; } + assert( pTabList->nSrc==1 ); - /* This function normally generates a nested loop for all tables in - ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should - ** only generate code for the first table in pTabList and assume that - ** any cursors associated with subsequent tables are uninitialized. + /* Locate the table which we want to update. */ - nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto update_cleanup; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - /* Allocate and initialize the WhereInfo structure that will become the - ** return value. A single allocation is used to store the WhereInfo - ** struct, the contents of WhereInfo.a[], the WhereClause structure - ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte - ** field (type Bitmask) it must be aligned on an 8-byte boundary on - ** some architectures. Hence the ROUND8() below. + /* Figure out if we have any triggers and if the table being + ** updated is a view. */ - db = pParse->db; - nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocZero(db, - nByteWInfo + - sizeof(WhereClause) + - sizeof(WhereMaskSet) - ); - if( db->mallocFailed ){ - sqlite3DbFree(db, pWInfo); - pWInfo = 0; - goto whereBeginError; +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask); + isView = pTab->pSelect!=0; + assert( pTrigger || tmask==0 ); +#else +# define pTrigger 0 +# define isView 0 +# define tmask 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto update_cleanup; } - pWInfo->nLevel = nTabList; - pWInfo->pParse = pParse; - pWInfo->pTabList = pTabList; - pWInfo->iBreak = sqlite3VdbeMakeLabel(v); - pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; - pWInfo->wctrlFlags = wctrlFlags; - pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWC[1]; + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto update_cleanup; + } + aXRef = sqlite3DbMallocRaw(db, sizeof(int) * pTab->nCol ); + if( aXRef==0 ) goto update_cleanup; + for(i=0; inCol; i++) aXRef[i] = -1; - /* Split the WHERE clause into separate subexpressions where each - ** subexpression is separated by an AND operator. - */ - initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet); - sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ - - /* Special case: a WHERE clause that is constant. Evaluate the - ** expression and either jump over all of the code or fall thru. + /* Allocate a cursors for the main database table and for all indices. + ** The index cursors might not be used, but if they are used they + ** need to occur right after the database cursor. So go ahead and + ** allocate enough space, just in case. */ - if( pWhere && (nTabList==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ - sqlite3ExprIfFalse(pParse, pWhere, pWInfo->iBreak, SQLITE_JUMPIFNULL); - pWhere = 0; + pTabList->a[0].iCursor = iCur = pParse->nTab++; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + pParse->nTab++; } - /* Assign a bit from the bitmask to every term in the FROM clause. - ** - ** When assigning bitmask values to FROM clause cursors, it must be - ** the case that if X is the bitmask for the N-th FROM clause term then - ** the bitmask for all FROM clause terms to the left of the N-th term - ** is (X-1). An expression from the ON clause of a LEFT JOIN can use - ** its Expr.iRightJoinTable value to find the bitmask of the right table - ** of the join. Subtracting one from the right table bitmask gives a - ** bitmask for all tables to the left of the join. Knowing the bitmask - ** for all tables to the left of a left join is important. Ticket #3015. - ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** - ** Note that bitmasks are created for all pTabList->nSrc tables in - ** pTabList, not just the first nTabList tables. nTabList is normally - ** equal to pTabList->nSrc but might be shortened to 1 if the - ** WHERE_ONETABLE_ONLY flag is set. + /* Initialize the name-context */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + + /* Resolve the column names in all the expressions of the + ** of the UPDATE statement. Also find the column index + ** for each column to be updated in the pChanges array. For each + ** column to be updated, make sure we have authorization to change + ** that column. */ - assert( pWC->vmask==0 && pMaskSet->n==0 ); - for(i=0; inSrc; i++){ - createMask(pMaskSet, pTabList->a[i].iCursor); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - pWC->vmask |= ((Bitmask)1 << i); + chngRowid = 0; + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ + goto update_cleanup; + } + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ){ + if( j==pTab->iPKey ){ + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + } + aXRef[j] = i; + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pChanges->a[i].zName) ){ + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + }else{ + sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); + pParse->checkSchema = 1; + goto update_cleanup; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int rc; + rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, + pTab->aCol[j].zName, db->aDb[iDb].zName); + if( rc==SQLITE_DENY ){ + goto update_cleanup; + }else if( rc==SQLITE_IGNORE ){ + aXRef[j] = -1; + } } #endif } -#ifndef NDEBUG - { - Bitmask toTheLeft = 0; - for(i=0; inSrc; i++){ - Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); - assert( (m-1)==toTheLeft ); - toTheLeft |= m; + + hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngRowid); + + /* Allocate memory for the array aRegIdx[]. There is one entry in the + ** array for each index associated with table being updated. Fill in + ** the value with a register number for indices that are to be used + ** and with zero for unused indices. + */ + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} + if( nIdx>0 ){ + aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx ); + if( aRegIdx==0 ) goto update_cleanup; + } + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int reg; + if( hasFK || chngRowid ){ + reg = ++pParse->nMem; + }else{ + reg = 0; + for(i=0; inColumn; i++){ + if( aXRef[pIdx->aiColumn[i]]>=0 ){ + reg = ++pParse->nMem; + break; + } + } } + aRegIdx[j] = reg; + } + + /* Begin generating code. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto update_cleanup; + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + pWhere = 0; + pTabList = 0; + goto update_cleanup; } #endif - /* Analyze all of the subexpressions. Note that exprAnalyze() might - ** add new virtual terms onto the end of the WHERE clause. We do not - ** want to analyze these virtual terms, so start analyzing at the end - ** and work forward so that the added virtual terms are never processed. + /* Allocate required registers. */ + regOldRowid = regNewRowid = ++pParse->nMem; + if( pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngRowid || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + + /* Start the view context. */ + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + } + + /* If we are trying to update a view, realize that view into + ** a ephemeral table. */ - exprAnalyzeAll(pTabList, pWC); - if( db->mallocFailed ){ - goto whereBeginError; +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( isView ){ + sqlite3MaterializeView(pParse, pTab, pWhere, iCur); } +#endif - /* Chose the best index to use for each table in the FROM clause. - ** - ** This loop fills in the following fields: - ** - ** pWInfo->a[].pIdx The index to use for this level of the loop. - ** pWInfo->a[].wsFlags WHERE_xxx flags associated with pIdx - ** pWInfo->a[].nEq The number of == and IN constraints - ** pWInfo->a[].iFrom Which term of the FROM clause is being coded - ** pWInfo->a[].iTabCur The VDBE cursor for the database table - ** pWInfo->a[].iIdxCur The VDBE cursor for the index - ** pWInfo->a[].pTerm When wsFlags==WO_OR, the OR-clause term - ** - ** This loop also figures out the nesting order of tables in the FROM - ** clause. + /* Resolve the column names in all the expressions in the + ** WHERE clause. */ - notReady = ~(Bitmask)0; - pTabItem = pTabList->a; - pLevel = pWInfo->a; - andFlags = ~0; - WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; iokOnePass; - /* Loop through the remaining entries in the FROM clause to find the - ** next nested loop. The loop tests all FROM clause entries - ** either once or twice. - ** - ** The first test is always performed if there are two or more entries - ** remaining and never performed if there is only one FROM clause entry - ** to choose from. The first test looks for an "optimal" scan. In - ** this context an optimal scan is one that uses the same strategy - ** for the given FROM clause entry as would be selected if the entry - ** were used as the innermost nested loop. In other words, a table - ** is chosen such that the cost of running that table cannot be reduced - ** by waiting for other tables to run first. This "optimal" test works - ** by first assuming that the FROM clause is on the inner loop and finding - ** its query plan, then checking to see if that query plan uses any - ** other FROM clause terms that are notReady. If no notReady terms are - ** used then the "optimal" query plan works. - ** - ** Note that the WhereCost.nRow parameter for an optimal scan might - ** not be as small as it would be if the table really were the innermost - ** join. The nRow value can be reduced by WHERE clause constraints - ** that do not use indices. But this nRow reduction only happens if the - ** table really is the innermost join. - ** - ** The second loop iteration is only performed if no optimal scan - ** strategies were found by the first iteration. This second iteration - ** is used to search for the lowest cost scan overall. - ** - ** Previous versions of SQLite performed only the second iteration - - ** the next outermost loop was always that with the lowest overall - ** cost. However, this meant that SQLite could select the wrong plan - ** for scripts such as the following: - ** - ** CREATE TABLE t1(a, b); - ** CREATE TABLE t2(c, d); - ** SELECT * FROM t2, t1 WHERE t2.rowid = t1.a; - ** - ** The best strategy is to iterate through table t1 first. However it - ** is not possible to determine this with a simple greedy algorithm. - ** Since the cost of a linear scan through table t2 is the same - ** as the cost of a linear scan through table t1, a simple greedy - ** algorithm may choose to use t2 for the outer loop, which is a much - ** costlier approach. - */ - nUnconstrained = 0; - notIndexed = 0; - for(isOptimal=(iFrom=0 && bestJ<0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ - for(j=iFrom, pTabItem=&pTabList->a[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; - if( j!=iFrom && doNotReorder ) break; - m = getMask(pMaskSet, pTabItem->iCursor); - if( (m & notReady)==0 ){ - if( j==iFrom ) iFrom++; - continue; - } - mask = (isOptimal ? m : notReady); - pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); - if( pTabItem->pIndex==0 ) nUnconstrained++; - - WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", - j, isOptimal)); - assert( pTabItem->pTab ); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTabItem->pTab) ){ - sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost, pp); - }else -#endif - { - bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost); - } - assert( isOptimal || (sCost.used¬Ready)==0 ); + /* Remember the rowid of every item to be updated. + */ + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regOldRowid); + if( !okOnePass ){ + regRowSet = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); + } - /* If an INDEXED BY clause is present, then the plan must use that - ** index if it uses any index at all */ - assert( pTabItem->pIndex==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sCost.plan.u.pIdx==pTabItem->pIndex ); + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); - if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - notIndexed |= m; - } + /* Initialize the count of updated rows + */ + if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } - /* Conditions under which this table becomes the best so far: - ** - ** (1) The table must not depend on other tables that have not - ** yet run. - ** - ** (2) A full-table-scan plan cannot supercede another plan unless - ** it is an "optimal" plan as defined above. - ** - ** (3) All tables have an INDEXED BY clause or this table lacks an - ** INDEXED BY clause or this table uses the specific - ** index specified by its INDEXED BY clause. This rule ensures - ** that a best-so-far is always selected even if an impossible - ** combination of INDEXED BY clauses are given. The error - ** will be detected and relayed back to the application later. - ** The NEVER() comes about because rule (2) above prevents - ** An indexable full-table-scan from reaching rule (3). - ** - ** (4) The plan cost must be lower than prior plans or else the - ** cost must be the same and the number of rows must be lower. - */ - if( (sCost.used¬Ready)==0 /* (1) */ - && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) - && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ - || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sCost.rCostpIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_Replace ){ + openAll = 1; + break; } - if( doNotReorder ) break; } } - assert( bestJ>=0 ); - assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%g and nRow=%g\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ - *ppOrderBy = 0; - } - andFlags &= bestPlan.plan.wsFlags; - pLevel->plan = bestPlan.plan; - testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); - testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); - if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ - pLevel->iIdxCur = pParse->nTab++; - }else{ - pLevel->iIdxCur = -1; - } - notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); - pLevel->iFrom = (u8)bestJ; - if( bestPlan.plan.nRow>=(double)1 ){ - pParse->nQueryLoop *= bestPlan.plan.nRow; - } - - /* Check that if the table scanned by this loop iteration had an - ** INDEXED BY clause attached to it, that the named index is being - ** used for the scan. If not, then query compilation has failed. - ** Return an error. - */ - pIdx = pTabList->a[bestJ].pIndex; - if( pIdx ){ - if( (bestPlan.plan.wsFlags & WHERE_INDEXED)==0 ){ - sqlite3ErrorMsg(pParse, "cannot use index: %s", pIdx->zName); - goto whereBeginError; - }else{ - /* If an INDEXED BY clause is used, the bestIndex() function is - ** guaranteed to find the index specified in the INDEXED BY clause - ** if it find an index at all. */ - assert( bestPlan.plan.u.pIdx==pIdx ); + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + if( openAll || aRegIdx[i]>0 ){ + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iCur+i+1, pIdx->tnum, iDb, + (char*)pKey, P4_KEYINFO_HANDOFF); + assert( pParse->nTab>iCur+i+1 ); } } } - WHERETRACE(("*** Optimizer Finished ***\n")); - if( pParse->nErr || db->mallocFailed ){ - goto whereBeginError; + + /* Top of the update loop */ + if( okOnePass ){ + int a1 = sqlite3VdbeAddOp1(v, OP_NotNull, regOldRowid); + addr = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, a1); + }else{ + addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, 0, regOldRowid); } - /* If the total query only selects a single row, then the ORDER BY - ** clause is irrelevant. - */ - if( (andFlags & WHERE_UNIQUE)!=0 && ppOrderBy ){ - *ppOrderBy = 0; + /* Make cursor iCur point to the record that is being updated. If + ** this record does not exist for some reason (deleted by a trigger, + ** for example, then jump to the next iteration of the RowSet loop. */ + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); + + /* If the record number will change, set register regNewRowid to + ** contain the new value. If the record number is not being modified, + ** then regNewRowid is the same register as regOldRowid, which is + ** already populated. */ + assert( chngRowid || pTrigger || hasFK || regOldRowid==regNewRowid ); + if( chngRowid ){ + sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); } - /* If the caller is an UPDATE or DELETE statement that is requesting - ** to use a one-pass algorithm, determine if this is appropriate. - ** The one-pass algorithm only works if the WHERE clause constraints - ** the statement to update a single row. - */ - assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_UNIQUE)!=0 ){ - pWInfo->okOnePass = 1; - pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; + /* If there are triggers on this table, populate an array of registers + ** with the required old.* column data. */ + if( hasFK || pTrigger ){ + u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); + oldmask |= sqlite3TriggerColmask(pParse, + pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError + ); + for(i=0; inCol; i++){ + if( aXRef[i]<0 || oldmask==0xffffffff || (i<32 && (oldmask & (1<nRowOut = (double)1; - for(i=0, pLevel=pWInfo->a; ia[pLevel->iFrom]; - pTab = pTabItem->pTab; - pLevel->iTabCur = pTabItem->iCursor; - pWInfo->nRowOut *= pLevel->plan.nRow; - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ - /* Do nothing */ - }else -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); - int iCur = pTabItem->iCursor; - sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); - }else -#endif - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OMIT_OPEN)==0 ){ - int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; - sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); - testcase( pTab->nCol==BMS-1 ); - testcase( pTab->nCol==BMS ); - if( !pWInfo->okOnePass && pTab->nColcolUsed; - int n = 0; - for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, - SQLITE_INT_TO_PTR(n), P4_INT32); - assert( n<=pTab->nCol ); - } + /* Populate the array of registers beginning at regNew with the new + ** row data. This array is used to check constaints, create the new + ** table and index records, and as the values for any new.* references + ** made by triggers. + ** + ** If there are one or more BEFORE triggers, then do not populate the + ** registers associated with columns that are (a) not modified by + ** this UPDATE statement and (b) not accessed by new.* references. The + ** values for registers not modified by the UPDATE must be reloaded from + ** the database after the BEFORE triggers are fired anyway (as the trigger + ** may have modified them). So not loading those that are not going to + ** be used eliminates some redundant opcodes. + */ + newmask = sqlite3TriggerColmask( + pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError + ); + for(i=0; inCol; i++){ + if( i==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); }else{ - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - } -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); - }else -#endif - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - Index *pIx = pLevel->plan.u.pIdx; - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIx); - int iIdxCur = pLevel->iIdxCur; - assert( pIx->pSchema==pTab->pSchema ); - assert( iIdxCur>=0 ); - sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIx->tnum, iDb, - (char*)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIx->zName)); + j = aXRef[i]; + if( j>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); + }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask&(1<pMaskSet, pTabItem->iCursor); } - pWInfo->iTop = sqlite3VdbeCurrentAddr(v); - if( db->mallocFailed ) goto whereBeginError; - /* Generate the code to do the search. Each iteration of the for - ** loop below generates code for a single nested loop of the VM - ** program. + /* Fire any BEFORE UPDATE triggers. This happens before constraints are + ** verified. One could argue that this is wrong. */ - notReady = ~(Bitmask)0; - for(i=0; ia[i]; - explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); - pWInfo->iContinue = pLevel->addrCont; - } + if( tmask&TRIGGER_BEFORE ){ + sqlite3VdbeAddOp2(v, OP_Affinity, regNew, pTab->nCol); + sqlite3TableAffinityStr(v, pTab); + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_BEFORE, pTab, regOldRowid, onError, addr); -#ifdef SQLITE_TEST /* For testing and debugging use only */ - /* Record in the query plan information about the current table - ** and the index used to access it (if any). If the table itself - ** is not used, its name is just '{}'. If no index is used - ** the index is listed as "{}". If the primary key is used the - ** index name is '*'. - */ - for(i=0; ia[i]; - pTabItem = &pTabList->a[pLevel->iFrom]; - z = pTabItem->zAlias; - if( z==0 ) z = pTabItem->pTab->zName; - n = sqlite3Strlen30(z); - if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){ - if( pLevel->plan.wsFlags & WHERE_IDX_ONLY ){ - memcpy(&sqlite3_query_plan[nQPlan], "{}", 2); - nQPlan += 2; - }else{ - memcpy(&sqlite3_query_plan[nQPlan], z, n); - nQPlan += n; - } - sqlite3_query_plan[nQPlan++] = ' '; - } - testcase( pLevel->plan.wsFlags & WHERE_ROWID_EQ ); - testcase( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ); - if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - memcpy(&sqlite3_query_plan[nQPlan], "* ", 2); - nQPlan += 2; - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - n = sqlite3Strlen30(pLevel->plan.u.pIdx->zName); - if( n+nQPlan < sizeof(sqlite3_query_plan)-2 ){ - memcpy(&sqlite3_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); - nQPlan += n; - sqlite3_query_plan[nQPlan++] = ' '; + /* The row-trigger may have deleted the row being updated. In this + ** case, jump to the next row. No updates or AFTER triggers are + ** required. This behaviour - what happens when the row being updated + ** is deleted or renamed by a BEFORE trigger - is left undefined in the + ** documentation. + */ + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); + + /* If it did not delete it, the row-trigger may still have modified + ** some of the columns of the row being updated. Load the values for + ** all columns not modified by the update statement into their + ** registers in case this has happened. + */ + for(i=0; inCol; i++){ + if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i); + sqlite3ColumnDefault(v, pTab, i, regNew+i); } - }else{ - memcpy(&sqlite3_query_plan[nQPlan], "{} ", 3); - nQPlan += 3; } } - while( nQPlan>0 && sqlite3_query_plan[nQPlan-1]==' ' ){ - sqlite3_query_plan[--nQPlan] = 0; - } - sqlite3_query_plan[nQPlan] = 0; - nQPlan = 0; -#endif /* SQLITE_TEST // Testing and debugging use only */ - /* Record the continuation address in the WhereInfo structure. Then - ** clean up and return. - */ - return pWInfo; + if( !isView ){ + int j1; /* Address of jump instruction */ - /* Jump here if malloc fails */ -whereBeginError: - if( pWInfo ){ - pParse->nQueryLoop = pWInfo->savedNQueryLoop; - whereInfoFree(db, pWInfo); - } - return 0; -} + /* Do constraint checks. */ + sqlite3GenerateConstraintChecks(pParse, pTab, iCur, regNewRowid, + aRegIdx, (chngRowid?regOldRowid:0), 1, onError, addr, 0); -/* -** Generate the end of the WHERE loop. See comments on -** sqlite3WhereBegin() for additional information. -*/ -SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ - Parse *pParse = pWInfo->pParse; - Vdbe *v = pParse->pVdbe; - int i; - WhereLevel *pLevel; - SrcList *pTabList = pWInfo->pTabList; - sqlite3 *db = pParse->db; + /* Do FK constraint checks. */ + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, regOldRowid, 0); + } - /* Generate loop termination code. - */ - sqlite3ExprCacheClear(pParse); - for(i=pWInfo->nLevel-1; i>=0; i--){ - pLevel = &pWInfo->a[i]; - sqlite3VdbeResolveLabel(v, pLevel->addrCont); - if( pLevel->op!=OP_Noop ){ - sqlite3VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); - sqlite3VdbeChangeP5(v, pLevel->p5); + /* Delete the index entries associated with the current record. */ + j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regOldRowid); + sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, aRegIdx); + + /* If changing the record number, delete the old record. */ + if( hasFK || chngRowid ){ + sqlite3VdbeAddOp2(v, OP_Delete, iCur, 0); } - if( pLevel->plan.wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ - struct InLoop *pIn; - int j; - sqlite3VdbeResolveLabel(v, pLevel->addrNxt); - for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ - sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); - sqlite3VdbeJumpHere(v, pIn->addrInTop-1); - } - sqlite3DbFree(db, pLevel->u.in.aInLoop); + sqlite3VdbeJumpHere(v, j1); + + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, 0, regNewRowid); } - sqlite3VdbeResolveLabel(v, pLevel->addrBrk); - if( pLevel->iLeftJoin ){ - int addr; - addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ); - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); - } - if( pLevel->iIdxCur>=0 ){ - sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); - } - if( pLevel->op==OP_Return ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); - }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); - } - sqlite3VdbeJumpHere(v, addr); + + /* Insert the new index entries and the new record. */ + sqlite3CompleteInsertion(pParse, pTab, iCur, regNewRowid, aRegIdx, 1, 0, 0); + + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just updated. */ + if( hasFK ){ + sqlite3FkActions(pParse, pTab, pChanges, regOldRowid); } } - /* The "break" point is here, just past the end of the outer loop. - ** Set it. + /* Increment the row counter */ - sqlite3VdbeResolveLabel(v, pWInfo->iBreak); + if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + } - /* Close all of the cursors that were opened by sqlite3WhereBegin. - */ - assert( pWInfo->nLevel==1 || pWInfo->nLevel==pTabList->nSrc ); - for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ - struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; - Table *pTab = pTabItem->pTab; - assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)==0 - && pTab->pSelect==0 - && (pWInfo->wctrlFlags & WHERE_OMIT_CLOSE)==0 - ){ - int ws = pLevel->plan.wsFlags; - if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); - } - if( (ws & WHERE_INDEXED)!=0 && (ws & WHERE_TEMP_INDEX)==0 ){ - sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); - } - } + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_AFTER, pTab, regOldRowid, onError, addr); - /* If this scan uses an index, make code substitutions to read data - ** from the index in preference to the table. Sometimes, this means - ** the table need never be read from. This is a performance boost, - ** as the vdbe level waits until the table is read before actually - ** seeking the table cursor to the record corresponding to the current - ** position in the index. - ** - ** Calls to the code generator in between sqlite3WhereBegin and - ** sqlite3WhereEnd will have created code that references the table - ** directly. This loop scans all that code looking for opcodes - ** that reference the table and converts them into opcodes that - ** reference the index. - */ - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){ - int k, j, last; - VdbeOp *pOp; - Index *pIdx = pLevel->plan.u.pIdx; + /* Repeat the above with the next record to be updated, until + ** all record selected by the WHERE clause have been updated. + */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeJumpHere(v, addr); - assert( pIdx!=0 ); - pOp = sqlite3VdbeGetOp(v, pWInfo->iTop); - last = sqlite3VdbeCurrentAddr(v); - for(k=pWInfo->iTop; kp1!=pLevel->iTabCur ) continue; - if( pOp->opcode==OP_Column ){ - for(j=0; jnColumn; j++){ - if( pOp->p2==pIdx->aiColumn[j] ){ - pOp->p2 = j; - pOp->p1 = pLevel->iIdxCur; - break; - } - } - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || jnColumn ); - }else if( pOp->opcode==OP_Rowid ){ - pOp->p1 = pLevel->iIdxCur; - pOp->opcode = OP_IdxRowid; - } - } + /* Close all tables */ + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + if( openAll || aRegIdx[i]>0 ){ + sqlite3VdbeAddOp2(v, OP_Close, iCur+i+1, 0); } } + sqlite3VdbeAddOp2(v, OP_Close, iCur, 0); - /* Final cleanup + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. */ - pParse->nQueryLoop = pWInfo->savedNQueryLoop; - whereInfoFree(db, pWInfo); + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows that were changed. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC); + } + +update_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3DbFree(db, aRegIdx); + sqlite3DbFree(db, aXRef); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pChanges); + sqlite3ExprDelete(db, pWhere); return; } +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** thely may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif -/************** End of where.c ***********************************************/ -/************** Begin file parse.c *******************************************/ -/* Driver template for the LEMON parser generator. -** The author disclaims copyright to this source code. +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Generate code for an UPDATE of a virtual table. ** -** This version of "lempar.c" is modified, slightly, for use by SQLite. -** The only modifications are the addition of a couple of NEVER() -** macros to disable tests that are needed in the case of a general -** LALR(1) grammar but which are always false in the -** specific grammar used by SQLite. +** The strategy is that we create an ephemerial table that contains +** for each row to be changed: +** +** (A) The original rowid of that row. +** (B) The revised rowid for the row. (note1) +** (C) The content of every column in the row. +** +** Then we loop over this ephemeral table and for each row in +** the ephermeral table call VUpdate. +** +** When finished, drop the ephemeral table. +** +** (note1) Actually, if we know in advance that (A) is always the same +** as (B) we only store (A), then duplicate (A) when pulling +** it out of the ephemeral table before calling VUpdate. */ -/* First off, code is included that follows the "include" declaration -** in the input grammar file. */ - +static void updateVirtualTable( + Parse *pParse, /* The parsing context */ + SrcList *pSrc, /* The virtual table to be modified */ + Table *pTab, /* The virtual table */ + ExprList *pChanges, /* The columns to change in the UPDATE statement */ + Expr *pRowid, /* Expression used to recompute the rowid */ + int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ +){ + Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ + ExprList *pEList = 0; /* The result set of the SELECT statement */ + Select *pSelect = 0; /* The SELECT statement */ + Expr *pExpr; /* Temporary expression */ + int ephemTab; /* Table holding the result of the SELECT */ + int i; /* Loop counter */ + int addr; /* Address of top of loop */ + int iReg; /* First register in set passed to OP_VUpdate */ + sqlite3 *db = pParse->db; /* Database connection */ + const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); + SelectDest dest; -/* -** Disable all error recovery processing in the parser push-down -** automaton. -*/ -#define YYNOERRORRECOVERY 1 + /* Construct the SELECT statement that will find the new values for + ** all updated rows. + */ + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); + if( pRowid ){ + pEList = sqlite3ExprListAppend(pParse, pEList, + sqlite3ExprDup(db, pRowid, 0)); + } + assert( pTab->iPKey<0 ); + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ){ + pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); + }else{ + pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); + } + pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); + } + pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); + + /* Create the ephemeral table into which the update results will + ** be stored. + */ + assert( v ); + ephemTab = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); -/* -** Make yytestcase() the same as testcase() -*/ -#define yytestcase(X) testcase(X) + /* fill the ephemeral table + */ + sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); + sqlite3Select(pParse, pSelect, &dest); -/* -** An instance of this structure holds information about the -** LIMIT clause of a SELECT statement. -*/ -struct LimitVal { - Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ - Expr *pOffset; /* The OFFSET expression. NULL if there is none */ -}; + /* Generate code to scan the ephemeral table and call VUpdate. */ + iReg = ++pParse->nMem; + pParse->nMem += pTab->nCol+1; + addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); + for(i=0; inCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + } + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); -/* -** An instance of this structure is used to store the LIKE, -** GLOB, NOT LIKE, and NOT GLOB operators. -*/ -struct LikeOp { - Token eOperator; /* "like" or "glob" or "regexp" */ - int not; /* True if the NOT keyword is present */ -}; + /* Cleanup */ + sqlite3SelectDelete(db, pSelect); +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ +/************** End of update.c **********************************************/ +/************** Begin file vacuum.c ******************************************/ /* -** An instance of the following structure describes the event of a -** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, -** TK_DELETE, or TK_INSTEAD. If the event is of the form +** 2003 April 6 ** -** UPDATE ON (a,b,c) +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Then the "b" IdList records the list "a,b,c". +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the VACUUM command. +** +** Most of the code in this file may be omitted by defining the +** SQLITE_OMIT_VACUUM macro. */ -struct TrigEvent { int a; IdList * b; }; +#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) /* -** An instance of this structure holds the ATTACH key and the key type. +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. */ -struct AttachKey { int type; Token key; }; +static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} +/* +** Execute zSql on database db. Return an error code. +*/ +static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + VVA_ONLY( int rc; ) + if( !zSql ){ + return SQLITE_NOMEM; + } + if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + return sqlite3_errcode(db); + } + VVA_ONLY( rc = ) sqlite3_step(pStmt); + assert( rc!=SQLITE_ROW ); + return vacuumFinalize(db, pStmt, pzErrMsg); +} - /* This is a utility routine used to set the ExprSpan.zStart and - ** ExprSpan.zEnd values of pOut so that the span covers the complete - ** range of text beginning with pStart and going to the end of pEnd. - */ - static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ - pOut->zStart = pStart->z; - pOut->zEnd = &pEnd->z[pEnd->n]; +/* +** Execute zSql on database db. The statement returns exactly +** one column. Execute this as SQL on the same database. +*/ +static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); + if( rc!=SQLITE_OK ){ + vacuumFinalize(db, pStmt, pzErrMsg); + return rc; + } } - /* Construct a new Expr object from a single identifier. Use the - ** new Expr to populate pOut. Set the span of pOut to be the identifier - ** that created the expression. - */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); - pOut->zStart = pValue->z; - pOut->zEnd = &pValue->z[pValue->n]; + return vacuumFinalize(db, pStmt, pzErrMsg); +} + +/* +** The non-standard VACUUM command is used to clean up the database, +** collapse free space, etc. It is modelled after the VACUUM command +** in PostgreSQL. +** +** In version 1.0.x of SQLite, the VACUUM command would call +** gdbm_reorganize() on all the database tables. But beginning +** with 2.0.0, SQLite no longer uses GDBM so this command has +** become a no-op. +*/ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); } + return; +} - /* This routine constructs a binary expression node out of two ExprSpan - ** objects and uses the result to populate a new ExprSpan object. +/* +** This routine implements the OP_Vacuum opcode of the VDBE. +*/ +SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ + int rc = SQLITE_OK; /* Return code from service routines */ + Btree *pMain; /* The database being vacuumed */ + Btree *pTemp; /* The temporary database we vacuum into */ + char *zSql = 0; /* SQL statements */ + int saved_flags; /* Saved value of the db->flags */ + int saved_nChange; /* Saved value of db->nChange */ + int saved_nTotalChange; /* Saved value of db->nTotalChange */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + Db *pDb = 0; /* Database to detach at end of vacuum */ + int isMemDb; /* True if vacuuming a :memory: database */ + int nRes; /* Bytes of reserved space at the end of each page */ + int nDb; /* Number of attached databases */ + + if( !db->autoCommit ){ + sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); + return SQLITE_ERROR; + } + if( db->activeVdbeCnt>1 ){ + sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); + return SQLITE_ERROR; + } + + /* Save the current value of the database flags so that it can be + ** restored before returning. Then set the writable-schema flag, and + ** disable CHECK and foreign key constraints. */ + saved_flags = db->flags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; + + pMain = db->aDb[0].pBt; + isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); + + /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + ** can be set to 'off' for this file, as it is not recovered if a crash + ** occurs anyway. The integrity of the database is maintained by a + ** (possibly synchronous) transaction opened on the main database before + ** sqlite3BtreeCopyFile() is called. + ** + ** An optimisation would be to use a non-journaled pager. + ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** that actually made the VACUUM run slower. Very little journalling + ** actually occurs when doing a vacuum since the vacuum_db is initially + ** empty. Only the journal header is written. Apparently it takes more + ** time to parse and run the PRAGMA to turn journalling off than it does + ** to write the journal header file. */ - static void spanBinaryExpr( - ExprSpan *pOut, /* Write the result here */ - Parse *pParse, /* The parsing context. Errors accumulate here */ - int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand */ - ExprSpan *pRight /* The right operand */ + nDb = db->nDb; + if( sqlite3TempInMemory(db) ){ + zSql = "ATTACH ':memory:' AS vacuum_db;"; + }else{ + zSql = "ATTACH '' AS vacuum_db;"; + } + rc = execSql(db, pzErrMsg, zSql); + if( db->nDb>nDb ){ + pDb = &db->aDb[db->nDb-1]; + assert( strcmp(pDb->zName,"vacuum_db")==0 ); + } + if( rc!=SQLITE_OK ) goto end_of_vacuum; + pTemp = db->aDb[db->nDb-1].pBt; + + /* The call to execSql() to attach the temp database has left the file + ** locked (as there was more than one active statement when the transaction + ** to read the schema was concluded. Unlock it here so that this doesn't + ** cause problems for the call to BtreeSetPageSize() below. */ + sqlite3BtreeCommit(pTemp); + + nRes = sqlite3BtreeGetReserve(pMain); + + /* A VACUUM cannot change the pagesize of an encrypted database. */ +#ifdef SQLITE_HAS_CODEC + if( db->nextPagesize ){ + extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); + int nKey; + char *zKey; + sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); + if( nKey ) db->nextPagesize = 0; + } +#endif + + /* Do not attempt to change the page size for a WAL database */ + if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) + ==PAGER_JOURNALMODE_WAL ){ + db->nextPagesize = 0; + } + + if( sqlite3BtreeSetPageSize(pTemp, sqlite3BtreeGetPageSize(pMain), nRes, 0) + || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) + || NEVER(db->mallocFailed) ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pOut->zStart = pLeft->zStart; - pOut->zEnd = pRight->zEnd; + rc = SQLITE_NOMEM; + goto end_of_vacuum; + } + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ){ + goto end_of_vacuum; } - /* Construct an expression node for a unary postfix operator +#ifndef SQLITE_OMIT_AUTOVACUUM + sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : + sqlite3BtreeGetAutoVacuum(pMain)); +#endif + + /* Begin a transaction */ + rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. */ - static void spanUnaryPostfix( - ExprSpan *pOut, /* Write the new expression node here */ - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ - Token *pPostOp /* The operand token for setting the span */ - ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pOperand->zStart; - pOut->zEnd = &pPostOp->z[pPostOp->n]; - } + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " + " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" + " AND rootpage>0" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" + " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " + " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* A routine to convert a binary TK_IS or TK_ISNOT expression into a - ** unary TK_ISNULL or TK_NOTNULL expression. */ - static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ - sqlite3 *db = pParse->db; - if( db->mallocFailed==0 && pY->op==TK_NULL ){ - pA->op = (u8)op; - sqlite3ExprDelete(db, pA->pRight); - pA->pRight = 0; + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + rc = execExecSql(db, pzErrMsg, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';'" + "FROM main.sqlite_master " + "WHERE type = 'table' AND name!='sqlite_sequence' " + " AND rootpage>0" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Copy over the sequence table + */ + rc = execExecSql(db, pzErrMsg, + "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " + "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execExecSql(db, pzErrMsg, + "SELECT 'INSERT INTO vacuum_db.' || quote(name) " + "|| ' SELECT * FROM main.' || quote(name) || ';' " + "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the SQLITE_MASTER table. + */ + rc = execSql(db, pzErrMsg, + "INSERT INTO vacuum_db.sqlite_master " + " SELECT type, name, tbl_name, rootpage, sql" + " FROM main.sqlite_master" + " WHERE type='view' OR type='trigger'" + " OR (type='table' AND rootpage=0)" + ); + if( rc ) goto end_of_vacuum; + + /* At this point, unless the main db was completely empty, there is now a + ** transaction open on the vacuum database, but not on the main database. + ** Open a btree level transaction on the main database. This allows a + ** call to sqlite3BtreeCopyFile(). The main database btree level + ** transaction is then committed, so the SQL level never knows it was + ** opened for writing. This way, the SQL transaction used to create the + ** temporary database never needs to be committed. + */ + { + u32 meta; + int i; + + /* This array determines which meta meta values are preserved in the + ** vacuum. Even entries are the meta value number and odd entries + ** are an increment to apply to the meta value after the vacuum. + ** The increment is used to increase the schema cookie so that other + ** connections to the same database will know to reread the schema. + */ + static const unsigned char aCopy[] = { + BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ + BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ + BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ + BTREE_USER_VERSION, 0, /* Preserve the user version */ + }; + + assert( 1==sqlite3BtreeIsInTrans(pTemp) ); + assert( 1==sqlite3BtreeIsInTrans(pMain) ); + + /* Copy Btree meta values */ + for(i=0; iflags */ + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; + sqlite3BtreeSetPageSize(pMain, -1, -1, 1); + + /* Currently there is an SQL level transaction open on the vacuum + ** database. No locks are held on any other files (since the main file + ** was committed at the btree level). So it safe to end the transaction + ** by manually setting the autoCommit flag to true and detaching the + ** vacuum database. The vacuum_db journal file is deleted when the pager + ** is closed by the DETACH. */ - static void spanUnaryPrefix( - ExprSpan *pOut, /* Write the new expression node here */ - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ - Token *pPreOp /* The operand token for setting the span */ - ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pPreOp->z; - pOut->zEnd = pOperand->zEnd; + db->autoCommit = 1; + + if( pDb ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; } -/* Next is all token values, in a form suitable for use by makeheaders. -** This section will be null unless lemon is run with the -m switch. -*/ -/* -** These constants (all generated automatically by the parser generator) -** specify the various kinds of tokens (terminals) that the parser -** understands. -** -** Each symbol here is a terminal symbol in the grammar. -*/ -/* Make sure the INTERFACE macro is defined. -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/* The next thing included is series of defines which control -** various aspects of the generated parser. -** YYCODETYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 terminals -** and nonterminals. "int" is used otherwise. -** YYNOCODE is a number of type YYCODETYPE which corresponds -** to no legal terminal or nonterminal number. This -** number is used to fill in empty slots of the hash -** table. -** YYFALLBACK If defined, this indicates that one or more tokens -** have fall-back values which should be used if the -** original value of the token will not parse. -** YYACTIONTYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 rules and -** states combined. "int" is used otherwise. -** sqlite3ParserTOKENTYPE is the data type used for minor tokens given -** directly to the parser from the tokenizer. -** YYMINORTYPE is the data type used for all minor tokens. -** This is typically a union of many types, one of -** which is sqlite3ParserTOKENTYPE. The entry in the union -** for base tokens is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument -** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument -** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser -** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -*/ -#define YYCODETYPE unsigned char -#define YYNOCODE 253 -#define YYACTIONTYPE unsigned short int -#define YYWILDCARD 67 -#define sqlite3ParserTOKENTYPE Token -typedef union { - int yyinit; - sqlite3ParserTOKENTYPE yy0; - int yy4; - struct TrigEvent yy90; - ExprSpan yy118; - TriggerStep* yy203; - u8 yy210; - struct {int value; int mask;} yy215; - SrcList* yy259; - struct LimitVal yy292; - Expr* yy314; - ExprList* yy322; - struct LikeOp yy342; - IdList* yy384; - Select* yy387; -} YYMINORTYPE; -#ifndef YYSTACKDEPTH -#define YYSTACKDEPTH 100 -#endif -#define sqlite3ParserARG_SDECL Parse *pParse; -#define sqlite3ParserARG_PDECL ,Parse *pParse -#define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse -#define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 630 -#define YYNRULE 329 -#define YYFALLBACK 1 -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) -/* The yyzerominor constant is used to initialize instances of -** YYMINORTYPE objects to zero. */ -static const YYMINORTYPE yyzerominor = { 0 }; + /* This both clears the schemas and reduces the size of the db->aDb[] + ** array. */ + sqlite3ResetInternalSchema(db, -1); -/* Define the yytestcase() macro to be a no-op if is not already defined -** otherwise. -** -** Applications can choose to define yytestcase() in the %include section -** to a macro that can assist in verifying code coverage. For production -** code the yytestcase() macro should be turned off. But it is useful -** for testing. -*/ -#ifndef yytestcase -# define yytestcase(X) -#endif + return rc; +} +#endif /* SQLITE_OMIT_VACUUM && SQLITE_OMIT_ATTACH */ -/* Next are the tables used to determine what action to take based on the -** current state and lookahead token. These tables are used to implement -** functions that take a state number and lookahead value and return an -** action integer. -** -** Suppose the action integer is N. Then the action is determined as -** follows -** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead -** token onto the stack and goto state N. -** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. -** -** N == YYNSTATE+YYNRULE A syntax error has occurred. -** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. -** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused -** slots in the yy_action[] table. -** -** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as -** -** yy_action[ yy_shift_ofst[S] + X ] -** -** If the index value yy_shift_ofst[S]+X is out of range or if the value -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] -** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table -** and that yy_default[S] should be used instead. +/************** End of vacuum.c **********************************************/ +/************** Begin file vtab.c ********************************************/ +/* +** 2006 June 10 ** -** The formula above is for computing the action when the lookahead is -** a terminal symbol. If the lookahead is a non-terminal (as occurs after -** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of -** YY_SHIFT_USE_DFLT. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** The following are the tables generated in this section: +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** yy_action[] A single table containing all actions. -** yy_lookahead[] A table containing the lookahead for each entry in -** yy_action. Used to detect hash collisions. -** yy_shift_ofst[] For each state, the offset into yy_action for -** shifting terminals. -** yy_reduce_ofst[] For each state, the offset into yy_action for -** shifting non-terminals after a reduce. -** yy_default[] Default action for each state. +************************************************************************* +** This file contains code used to help implement virtual tables. */ -#define YY_ACTTAB_COUNT (1557) -static const YYACTIONTYPE yy_action[] = { - /* 0 */ 313, 960, 186, 419, 2, 172, 627, 597, 55, 55, - /* 10 */ 55, 55, 48, 53, 53, 53, 53, 52, 52, 51, +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Before a virtual table xCreate() or xConnect() method is invoked, the +** sqlite3.pVtabCtx member variable is set to point to an instance of +** this struct allocated on the stack. It is used by the implementation of +** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which +** are invoked only from within xCreate and xConnect methods. +*/ +struct VtabCtx { + Table *pTab; + VTable *pVTable; +}; + +/* +** The actual function that does the work of creating a new module. +** This function implements the sqlite3_create_module() and +** sqlite3_create_module_v2() interfaces. +*/ +static int createModule( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + int rc, nName; + Module *pMod; + + sqlite3_mutex_enter(db->mutex); + nName = sqlite3Strlen30(zName); + pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); + if( pMod ){ + Module *pDel; + char *zCopy = (char *)(&pMod[1]); + memcpy(zCopy, zName, nName+1); + pMod->zName = zCopy; + pMod->pModule = pModule; + pMod->pAux = pAux; + pMod->xDestroy = xDestroy; + pDel = (Module *)sqlite3HashInsert(&db->aModule, zCopy, nName, (void*)pMod); + if( pDel && pDel->xDestroy ){ + sqlite3ResetInternalSchema(db, -1); + pDel->xDestroy(pDel->pAux); + } + sqlite3DbFree(db, pDel); + if( pDel==pMod ){ + db->mallocFailed = 1; + } + }else if( xDestroy ){ + xDestroy(pAux); + } + rc = sqlite3ApiExit(db, SQLITE_OK); + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** External API function used to create a new virtual-table module. +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux /* Context pointer for xCreate/xConnect */ +){ + return createModule(db, zName, pModule, pAux, 0); +} + +/* +** External API function used to create a new virtual-table module. +*/ +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + return createModule(db, zName, pModule, pAux, xDestroy); +} + +/* +** Lock the virtual table so that it cannot be disconnected. +** Locks nest. Every lock should have a corresponding unlock. +** If an unlock is omitted, resources leaks will occur. +** +** If a disconnect is attempted while a virtual table is locked, +** the disconnect is deferred until all locks have been removed. +*/ +SQLITE_PRIVATE void sqlite3VtabLock(VTable *pVTab){ + pVTab->nRef++; +} + + +/* +** pTab is a pointer to a Table structure representing a virtual-table. +** Return a pointer to the VTable object used by connection db to access +** this virtual-table, if one has been created, or NULL otherwise. +*/ +SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3 *db, Table *pTab){ + VTable *pVtab; + assert( IsVirtual(pTab) ); + for(pVtab=pTab->pVTable; pVtab && pVtab->db!=db; pVtab=pVtab->pNext); + return pVtab; +} + +/* +** Decrement the ref-count on a virtual table object. When the ref-count +** reaches zero, call the xDisconnect() method to delete the object. +*/ +SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ + sqlite3 *db = pVTab->db; + + assert( db ); + assert( pVTab->nRef>0 ); + assert( sqlite3SafetyCheckOk(db) ); + + pVTab->nRef--; + if( pVTab->nRef==0 ){ + sqlite3_vtab *p = pVTab->pVtab; + if( p ){ + p->pModule->xDisconnect(p); + } + sqlite3DbFree(db, pVTab); + } +} + +/* +** Table p is a virtual table. This function moves all elements in the +** p->pVTable list to the sqlite3.pDisconnect lists of their associated +** database connections to be disconnected at the next opportunity. +** Except, if argument db is not NULL, then the entry associated with +** connection db is left in the p->pVTable list. +*/ +static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ + VTable *pRet = 0; + VTable *pVTable = p->pVTable; + p->pVTable = 0; + + /* Assert that the mutex (if any) associated with the BtShared database + ** that contains table p is held by the caller. See header comments + ** above function sqlite3VtabUnlockList() for an explanation of why + ** this makes it safe to access the sqlite3.pDisconnect list of any + ** database connection that may have an entry in the p->pVTable list. + */ + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + + while( pVTable ){ + sqlite3 *db2 = pVTable->db; + VTable *pNext = pVTable->pNext; + assert( db2 ); + if( db2==db ){ + pRet = pVTable; + p->pVTable = pRet; + pRet->pNext = 0; + }else{ + pVTable->pNext = db2->pDisconnect; + db2->pDisconnect = pVTable; + } + pVTable = pNext; + } + + assert( !db || pRet ); + return pRet; +} + + +/* +** Disconnect all the virtual table objects in the sqlite3.pDisconnect list. +** +** This function may only be called when the mutexes associated with all +** shared b-tree databases opened using connection db are held by the +** caller. This is done to protect the sqlite3.pDisconnect list. The +** sqlite3.pDisconnect list is accessed only as follows: +** +** 1) By this function. In this case, all BtShared mutexes and the mutex +** associated with the database handle itself must be held. +** +** 2) By function vtabDisconnectAll(), when it adds a VTable entry to +** the sqlite3.pDisconnect list. In this case either the BtShared mutex +** associated with the database the virtual table is stored in is held +** or, if the virtual table is stored in a non-sharable database, then +** the database handle mutex is held. +** +** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously +** by multiple threads. It is thread-safe. +*/ +SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ + VTable *p = db->pDisconnect; + db->pDisconnect = 0; + + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3_mutex_held(db->mutex) ); + + if( p ){ + sqlite3ExpirePreparedStatements(db); + do { + VTable *pNext = p->pNext; + sqlite3VtabUnlock(p); + p = pNext; + }while( p ); + } +} + +/* +** Clear any and all virtual-table information from the Table record. +** This routine is called, for example, just before deleting the Table +** record. +** +** Since it is a virtual-table, the Table structure contains a pointer +** to the head of a linked list of VTable structures. Each VTable +** structure is associated with a single sqlite3* user of the schema. +** The reference count of the VTable structure associated with database +** connection db is decremented immediately (which may lead to the +** structure being xDisconnected and free). Any other VTable structures +** in the list are moved to the sqlite3.pDisconnect list of the associated +** database connection. +*/ +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ + if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + if( p->azModuleArg ){ + int i; + for(i=0; inModuleArg; i++){ + sqlite3DbFree(db, p->azModuleArg[i]); + } + sqlite3DbFree(db, p->azModuleArg); + } +} + +/* +** Add a new module argument to pTable->azModuleArg[]. +** The string is not copied - the pointer is stored. The +** string will be freed automatically when the table is +** deleted. +*/ +static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ + int i = pTable->nModuleArg++; + int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + char **azModuleArg; + azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); + if( azModuleArg==0 ){ + int j; + for(j=0; jazModuleArg[j]); + } + sqlite3DbFree(db, zArg); + sqlite3DbFree(db, pTable->azModuleArg); + pTable->nModuleArg = 0; + }else{ + azModuleArg[i] = zArg; + azModuleArg[i+1] = 0; + } + pTable->azModuleArg = azModuleArg; +} + +/* +** The parser calls this routine when it first sees a CREATE VIRTUAL TABLE +** statement. The module name has been parsed, but the optional list +** of parameters that follow the module name are still pending. +*/ +SQLITE_PRIVATE void sqlite3VtabBeginParse( + Parse *pParse, /* Parsing context */ + Token *pName1, /* Name of new table, or database name */ + Token *pName2, /* Name of new table or NULL */ + Token *pModuleName /* Name of the module for the virtual table */ +){ + int iDb; /* The database the table is being created in */ + Table *pTable; /* The new virtual table */ + sqlite3 *db; /* Database connection */ + + sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, 0); + pTable = pParse->pNewTable; + if( pTable==0 ) return; + assert( 0==pTable->pIndex ); + + db = pParse->db; + iDb = sqlite3SchemaToIndex(db, pTable->pSchema); + assert( iDb>=0 ); + + pTable->tabFlags |= TF_Virtual; + pTable->nModuleArg = 0; + addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(db, pTable, sqlite3DbStrDup(db, db->aDb[iDb].zName)); + addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); + pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Creating a virtual table invokes the authorization callback twice. + ** The first invocation, to obtain permission to INSERT a row into the + ** sqlite_master table, has already been made by sqlite3StartTable(). + ** The second call, to obtain permission to create the table, is made now. + */ + if( pTable->azModuleArg ){ + sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, + pTable->azModuleArg[0], pParse->db->aDb[iDb].zName); + } +#endif +} + +/* +** This routine takes the module argument that has been accumulating +** in pParse->zArg[] and appends it to the list of arguments on the +** virtual table currently under construction in pParse->pTable. +*/ +static void addArgumentToVtab(Parse *pParse){ + if( pParse->sArg.z && ALWAYS(pParse->pNewTable) ){ + const char *z = (const char*)pParse->sArg.z; + int n = pParse->sArg.n; + sqlite3 *db = pParse->db; + addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + } +} + +/* +** The parser calls this routine after the CREATE VIRTUAL TABLE statement +** has been completely parsed. +*/ +SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ + Table *pTab = pParse->pNewTable; /* The table being constructed */ + sqlite3 *db = pParse->db; /* The database connection */ + + if( pTab==0 ) return; + addArgumentToVtab(pParse); + pParse->sArg.z = 0; + if( pTab->nModuleArg<1 ) return; + + /* If the CREATE VIRTUAL TABLE statement is being entered for the + ** first time (in other words if the virtual table is actually being + ** created now instead of just being read out of sqlite_master) then + ** do additional initialization work and store the statement text + ** in the sqlite_master table. + */ + if( !db->init.busy ){ + char *zStmt; + char *zWhere; + int iDb; + Vdbe *v; + + /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ + if( pEnd ){ + pParse->sNameToken.n = (int)(pEnd->z - pParse->sNameToken.z) + pEnd->n; + } + zStmt = sqlite3MPrintf(db, "CREATE VIRTUAL TABLE %T", &pParse->sNameToken); + + /* A slot for the record has already been allocated in the + ** SQLITE_MASTER table. We just need to update that slot with all + ** the information we've collected. + ** + ** The VM register number pParse->regRowid holds the rowid of an + ** entry in the sqlite_master table tht was created for this vtab + ** by sqlite3StartTable(). + */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3NestedParse(pParse, + "UPDATE %Q.%s " + "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " + "WHERE rowid=#%d", + db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + pTab->zName, + pTab->zName, + zStmt, + pParse->regRowid + ); + sqlite3DbFree(db, zStmt); + v = sqlite3GetVdbe(pParse); + sqlite3ChangeCookie(pParse, iDb); + + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); + sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, + pTab->zName, sqlite3Strlen30(pTab->zName) + 1); + } + + /* If we are rereading the sqlite_master table create the in-memory + ** record of the table. The xConnect() method is not called until + ** the first time the virtual table is used in an SQL statement. This + ** allows a schema that contains virtual tables to be loaded before + ** the required virtual table implementations are registered. */ + else { + Table *pOld; + Schema *pSchema = pTab->pSchema; + const char *zName = pTab->zName; + int nName = sqlite3Strlen30(zName); + assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); + if( pOld ){ + db->mallocFailed = 1; + assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ + return; + } + pParse->pNewTable = 0; + } +} + +/* +** The parser calls this routine when it sees the first token +** of an argument to the module name in a CREATE VIRTUAL TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3VtabArgInit(Parse *pParse){ + addArgumentToVtab(pParse); + pParse->sArg.z = 0; + pParse->sArg.n = 0; +} + +/* +** The parser calls this routine for each token after the first token +** in an argument to the module name in a CREATE VIRTUAL TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ + Token *pArg = &pParse->sArg; + if( pArg->z==0 ){ + pArg->z = p->z; + pArg->n = p->n; + }else{ + assert(pArg->z < p->z); + pArg->n = (int)(&p->z[p->n] - pArg->z); + } +} + +/* +** Invoke a virtual table constructor (either xCreate or xConnect). The +** pointer to the function to invoke is passed as the fourth parameter +** to this procedure. +*/ +static int vtabCallConstructor( + sqlite3 *db, + Table *pTab, + Module *pMod, + int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), + char **pzErr +){ + VtabCtx sCtx; + VTable *pVTable; + int rc; + const char *const*azArg = (const char *const*)pTab->azModuleArg; + int nArg = pTab->nModuleArg; + char *zErr = 0; + char *zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + + if( !zModuleName ){ + return SQLITE_NOMEM; + } + + pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); + if( !pVTable ){ + sqlite3DbFree(db, zModuleName); + return SQLITE_NOMEM; + } + pVTable->db = db; + pVTable->pMod = pMod; + + /* Invoke the virtual table constructor */ + assert( &db->pVtabCtx ); + assert( xConstruct ); + sCtx.pTab = pTab; + sCtx.pVTable = pVTable; + db->pVtabCtx = &sCtx; + rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + db->pVtabCtx = 0; + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + + if( SQLITE_OK!=rc ){ + if( zErr==0 ){ + *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); + }else { + *pzErr = sqlite3MPrintf(db, "%s", zErr); + sqlite3_free(zErr); + } + sqlite3DbFree(db, pVTable); + }else if( ALWAYS(pVTable->pVtab) ){ + /* Justification of ALWAYS(): A correct vtab constructor must allocate + ** the sqlite3_vtab object if successful. */ + pVTable->pVtab->pModule = pMod->pModule; + pVTable->nRef = 1; + if( sCtx.pTab ){ + const char *zFormat = "vtable constructor did not declare schema: %s"; + *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + sqlite3VtabUnlock(pVTable); + rc = SQLITE_ERROR; + }else{ + int iCol; + /* If everything went according to plan, link the new VTable structure + ** into the linked list headed by pTab->pVTable. Then loop through the + ** columns of the table to see if any of them contain the token "hidden". + ** If so, set the Column.isHidden flag and remove the token from + ** the type string. */ + pVTable->pNext = pTab->pVTable; + pTab->pVTable = pVTable; + + for(iCol=0; iColnCol; iCol++){ + char *zType = pTab->aCol[iCol].zType; + int nType; + int i = 0; + if( !zType ) continue; + nType = sqlite3Strlen30(zType); + if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){ + for(i=0; i0 ){ + assert(zType[i-1]==' '); + zType[i-1] = '\0'; + } + pTab->aCol[iCol].isHidden = 1; + } + } + } + } + + sqlite3DbFree(db, zModuleName); + return rc; +} + +/* +** This function is invoked by the parser to call the xConnect() method +** of the virtual table pTab. If an error occurs, an error code is returned +** and an error left in pParse. +** +** This call is a no-op if table pTab is not a virtual table. +*/ +SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ + sqlite3 *db = pParse->db; + const char *zMod; + Module *pMod; + int rc; + + assert( pTab ); + if( (pTab->tabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){ + return SQLITE_OK; + } + + /* Locate the required virtual table module */ + zMod = pTab->azModuleArg[0]; + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + + if( !pMod ){ + const char *zModule = pTab->azModuleArg[0]; + sqlite3ErrorMsg(pParse, "no such module: %s", zModule); + rc = SQLITE_ERROR; + }else{ + char *zErr = 0; + rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + } + sqlite3DbFree(db, zErr); + } + + return rc; +} +/* +** Grow the db->aVTrans[] array so that there is room for at least one +** more v-table. Return SQLITE_NOMEM if a malloc fails, or SQLITE_OK otherwise. +*/ +static int growVTrans(sqlite3 *db){ + const int ARRAY_INCR = 5; + + /* Grow the sqlite3.aVTrans array if required */ + if( (db->nVTrans%ARRAY_INCR)==0 ){ + VTable **aVTrans; + int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); + aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); + if( !aVTrans ){ + return SQLITE_NOMEM; + } + memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); + db->aVTrans = aVTrans; + } + + return SQLITE_OK; +} + +/* +** Add the virtual table pVTab to the array sqlite3.aVTrans[]. Space should +** have already been reserved using growVTrans(). +*/ +static void addToVTrans(sqlite3 *db, VTable *pVTab){ + /* Add pVtab to the end of sqlite3.aVTrans */ + db->aVTrans[db->nVTrans++] = pVTab; + sqlite3VtabLock(pVTab); +} + +/* +** This function is invoked by the vdbe to call the xCreate method +** of the virtual table named zTab in database iDb. +** +** If an error occurs, *pzErr is set to point an an English language +** description of the error and an SQLITE_XXX error code is returned. +** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. +*/ +SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ + int rc = SQLITE_OK; + Table *pTab; + Module *pMod; + const char *zMod; + + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); + assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); + + /* Locate the required virtual table module */ + zMod = pTab->azModuleArg[0]; + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + + /* If the module has been registered and includes a Create method, + ** invoke it now. If the module has not been registered, return an + ** error. Otherwise, do nothing. + */ + if( !pMod ){ + *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); + rc = SQLITE_ERROR; + }else{ + rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xCreate, pzErr); + } + + /* Justification of ALWAYS(): The xConstructor method is required to + ** create a valid sqlite3_vtab if it returns SQLITE_OK. */ + if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + addToVTrans(db, sqlite3GetVTable(db, pTab)); + } + } + + return rc; +} + +/* +** This function is used to set the schema of a virtual table. It is only +** valid to call this function from within the xCreate() or xConnect() of a +** virtual table module. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ + Parse *pParse; + + int rc = SQLITE_OK; + Table *pTab; + char *zErr = 0; + + sqlite3_mutex_enter(db->mutex); + if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ + sqlite3Error(db, SQLITE_MISUSE, 0); + sqlite3_mutex_leave(db->mutex); + return SQLITE_MISUSE_BKPT; + } + assert( (pTab->tabFlags & TF_Virtual)!=0 ); + + pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); + if( pParse==0 ){ + rc = SQLITE_NOMEM; + }else{ + pParse->declareVtab = 1; + pParse->db = db; + pParse->nQueryLoop = 1; + + if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) + && pParse->pNewTable + && !db->mallocFailed + && !pParse->pNewTable->pSelect + && (pParse->pNewTable->tabFlags & TF_Virtual)==0 + ){ + if( !pTab->aCol ){ + pTab->aCol = pParse->pNewTable->aCol; + pTab->nCol = pParse->pNewTable->nCol; + pParse->pNewTable->nCol = 0; + pParse->pNewTable->aCol = 0; + } + db->pVtabCtx->pTab = 0; + }else{ + sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + rc = SQLITE_ERROR; + } + pParse->declareVtab = 0; + + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + sqlite3StackFree(db, pParse); + } + + assert( (rc&0xff)==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is invoked by the vdbe to call the xDestroy method +** of the virtual table named zTab in database iDb. This occurs +** when a DROP TABLE is mentioned. +** +** This call is a no-op if zTab is not a virtual table. +*/ +SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ + int rc = SQLITE_OK; + Table *pTab; + + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); + if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ + VTable *p = vtabDisconnectAll(db, pTab); + + assert( rc==SQLITE_OK ); + rc = p->pMod->pModule->xDestroy(p->pVtab); + + /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ + if( rc==SQLITE_OK ){ + assert( pTab->pVTable==p && p->pNext==0 ); + p->pVtab = 0; + pTab->pVTable = 0; + sqlite3VtabUnlock(p); + } + } + + return rc; +} + +/* +** This function invokes either the xRollback or xCommit method +** of each of the virtual tables in the sqlite3.aVTrans array. The method +** called is identified by the second argument, "offset", which is +** the offset of the method to call in the sqlite3_module structure. +** +** The array is cleared after invoking the callbacks. +*/ +static void callFinaliser(sqlite3 *db, int offset){ + int i; + if( db->aVTrans ){ + for(i=0; inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + sqlite3_vtab *p = pVTab->pVtab; + if( p ){ + int (*x)(sqlite3_vtab *); + x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset); + if( x ) x(p); + } + pVTab->iSavepoint = 0; + sqlite3VtabUnlock(pVTab); + } + sqlite3DbFree(db, db->aVTrans); + db->nVTrans = 0; + db->aVTrans = 0; + } +} + +/* +** Invoke the xSync method of all virtual tables in the sqlite3.aVTrans +** array. Return the error code for the first error that occurs, or +** SQLITE_OK if all xSync operations are successful. +** +** Set *pzErrmsg to point to a buffer that should be released using +** sqlite3DbFree() containing an error message, if one is available. +*/ +SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ + int i; + int rc = SQLITE_OK; + VTable **aVTrans = db->aVTrans; + + db->aVTrans = 0; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + int (*x)(sqlite3_vtab *); + sqlite3_vtab *pVtab = aVTrans[i]->pVtab; + if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ + rc = x(pVtab); + sqlite3DbFree(db, *pzErrmsg); + *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + } + } + db->aVTrans = aVTrans; + return rc; +} + +/* +** Invoke the xRollback method of all virtual tables in the +** sqlite3.aVTrans array. Then clear the array itself. +*/ +SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db){ + callFinaliser(db, offsetof(sqlite3_module,xRollback)); + return SQLITE_OK; +} + +/* +** Invoke the xCommit method of all virtual tables in the +** sqlite3.aVTrans array. Then clear the array itself. +*/ +SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db){ + callFinaliser(db, offsetof(sqlite3_module,xCommit)); + return SQLITE_OK; +} + +/* +** If the virtual table pVtab supports the transaction interface +** (xBegin/xRollback/xCommit and optionally xSync) and a transaction is +** not currently open, invoke the xBegin method now. +** +** If the xBegin call is successful, place the sqlite3_vtab pointer +** in the sqlite3.aVTrans array. +*/ +SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ + int rc = SQLITE_OK; + const sqlite3_module *pModule; + + /* Special case: If db->aVTrans is NULL and db->nVTrans is greater + ** than zero, then this function is being called from within a + ** virtual module xSync() callback. It is illegal to write to + ** virtual module tables in this case, so return SQLITE_LOCKED. + */ + if( sqlite3VtabInSync(db) ){ + return SQLITE_LOCKED; + } + if( !pVTab ){ + return SQLITE_OK; + } + pModule = pVTab->pVtab->pModule; + + if( pModule->xBegin ){ + int i; + + /* If pVtab is already in the aVTrans array, return early */ + for(i=0; inVTrans; i++){ + if( db->aVTrans[i]==pVTab ){ + return SQLITE_OK; + } + } + + /* Invoke the xBegin method. If successful, add the vtab to the + ** sqlite3.aVTrans[] array. */ + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + rc = pModule->xBegin(pVTab->pVtab); + if( rc==SQLITE_OK ){ + addToVTrans(db, pVTab); + } + } + } + return rc; +} + +/* +** Invoke either the xSavepoint, xRollbackTo or xRelease method of all +** virtual tables that currently have an open transaction. Pass iSavepoint +** as the second argument to the virtual table method invoked. +** +** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with +** an open transaction is invoked. +** +** If any virtual table method returns an error code other than SQLITE_OK, +** processing is abandoned and the error returned to the caller of this +** function immediately. If all calls to virtual table methods are successful, +** SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ + int rc = SQLITE_OK; + + assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); + assert( iSavepoint>=0 ); + if( db->aVTrans ){ + int i; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + const sqlite3_module *pMod = pVTab->pMod->pModule; + if( pMod->iVersion>=2 ){ + int (*xMethod)(sqlite3_vtab *, int); + switch( op ){ + case SAVEPOINT_BEGIN: + xMethod = pMod->xSavepoint; + pVTab->iSavepoint = iSavepoint+1; + break; + case SAVEPOINT_ROLLBACK: + xMethod = pMod->xRollbackTo; + break; + default: + xMethod = pMod->xRelease; + break; + } + if( xMethod && pVTab->iSavepoint>iSavepoint ){ + rc = xMethod(db->aVTrans[i]->pVtab, iSavepoint); + } + } + } + } + return rc; +} + +/* +** The first parameter (pDef) is a function implementation. The +** second parameter (pExpr) is the first argument to this function. +** If pExpr is a column in a virtual table, then let the virtual +** table implementation have an opportunity to overload the function. +** +** This routine is used to allow virtual table implementations to +** overload MATCH, LIKE, GLOB, and REGEXP operators. +** +** Return either the pDef argument (indicating no change) or a +** new FuncDef structure that is marked as ephemeral using the +** SQLITE_FUNC_EPHEM flag. +*/ +SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( + sqlite3 *db, /* Database connection for reporting malloc problems */ + FuncDef *pDef, /* Function to possibly overload */ + int nArg, /* Number of arguments to the function */ + Expr *pExpr /* First argument to the function */ +){ + Table *pTab; + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void *pArg = 0; + FuncDef *pNew; + int rc = 0; + char *zLowerName; + unsigned char *z; + + + /* Check to see the left operand is a column in a virtual table */ + if( NEVER(pExpr==0) ) return pDef; + if( pExpr->op!=TK_COLUMN ) return pDef; + pTab = pExpr->pTab; + if( NEVER(pTab==0) ) return pDef; + if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef; + pVtab = sqlite3GetVTable(db, pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction==0 ) return pDef; + + /* Call the xFindFunction method on the virtual table implementation + ** to see if the implementation wants to overload this function + */ + zLowerName = sqlite3DbStrDup(db, pDef->zName); + if( zLowerName ){ + for(z=(unsigned char*)zLowerName; *z; z++){ + *z = sqlite3UpperToLower[*z]; + } + rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xFunc, &pArg); + sqlite3DbFree(db, zLowerName); + } + if( rc==0 ){ + return pDef; + } + + /* Create a new ephemeral function definition for the overloaded + ** function */ + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) + + sqlite3Strlen30(pDef->zName) + 1); + if( pNew==0 ){ + return pDef; + } + *pNew = *pDef; + pNew->zName = (char *)&pNew[1]; + memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1); + pNew->xFunc = xFunc; + pNew->pUserData = pArg; + pNew->flags |= SQLITE_FUNC_EPHEM; + return pNew; +} + +/* +** Make sure virtual table pTab is contained in the pParse->apVirtualLock[] +** array so that an OP_VBegin will get generated for it. Add pTab to the +** array if it is missing. If pTab is already in the array, this routine +** is a no-op. +*/ +SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + int i, n; + Table **apVtabLock; + + assert( IsVirtual(pTab) ); + for(i=0; inVtabLock; i++){ + if( pTab==pToplevel->apVtabLock[i] ) return; + } + n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); + apVtabLock = sqlite3_realloc(pToplevel->apVtabLock, n); + if( apVtabLock ){ + pToplevel->apVtabLock = apVtabLock; + pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; + }else{ + pToplevel->db->mallocFailed = 1; + } +} + +/* +** Return the ON CONFLICT resolution mode in effect for the virtual +** table update operation currently in progress. +** +** The results of this routine are undefined unless it is called from +** within an xUpdate method. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ + static const unsigned char aMap[] = { + SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE + }; + assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); + assert( OE_Ignore==4 && OE_Replace==5 ); + assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); + return (int)aMap[db->vtabOnConflict-1]; +} + +/* +** Call from within the xCreate() or xConnect() methods to provide +** the SQLite core with additional information about the behavior +** of the virtual table being implemented. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { + VtabCtx *p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + p->pVTable->bConstraint = (u8)va_arg(ap, int); + } + break; + } + default: + rc = SQLITE_MISUSE_BKPT; + break; + } + va_end(ap); + + if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/************** End of vtab.c ************************************************/ +/************** Begin file where.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". +*/ + + +/* +** Trace output macros +*/ +#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3WhereTrace = 0; +#endif +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +# define WHERETRACE(X) if(sqlite3WhereTrace) sqlite3DebugPrintf X +#else +# define WHERETRACE(X) +#endif + +/* Forward reference +*/ +typedef struct WhereClause WhereClause; +typedef struct WhereMaskSet WhereMaskSet; +typedef struct WhereOrInfo WhereOrInfo; +typedef struct WhereAndInfo WhereAndInfo; +typedef struct WhereCost WhereCost; + +/* +** The query generator uses an array of instances of this structure to +** help it analyze the subexpressions of the WHERE clause. Each WHERE +** clause subexpression is separated from the others by AND operators, +** usually, or sometimes subexpressions separated by OR. +** +** All WhereTerms are collected into a single WhereClause structure. +** The following identity holds: +** +** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm +** +** When a term is of the form: +** +** X +** +** where X is a column name and is one of certain operators, +** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the +** cursor number and column number for X. WhereTerm.eOperator records +** the using a bitmask encoding defined by WO_xxx below. The +** use of a bitmask encoding for the operator allows us to search +** quickly for terms that match any of several different operators. +** +** A WhereTerm might also be two or more subterms connected by OR: +** +** (t1.X ) OR (t1.Y ) OR .... +** +** In this second case, wtFlag as the TERM_ORINFO set and eOperator==WO_OR +** and the WhereTerm.u.pOrInfo field points to auxiliary information that +** is collected about the +** +** If a term in the WHERE clause does not match either of the two previous +** categories, then eOperator==0. The WhereTerm.pExpr field is still set +** to the original subexpression content and wtFlags is set up appropriately +** but no other fields in the WhereTerm object are meaningful. +** +** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers, +** but they do so indirectly. A single WhereMaskSet structure translates +** cursor number into bits and the translated bit is stored in the prereq +** fields. The translation is used in order to maximize the number of +** bits that will fit in a Bitmask. The VDBE cursor numbers might be +** spread out over the non-negative integers. For example, the cursor +** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45. The WhereMaskSet +** translates these sparse cursor numbers into consecutive integers +** beginning with 0 in order to make the best possible use of the available +** bits in the Bitmask. So, in the example above, the cursor numbers +** would be mapped into integers 0 through 7. +** +** The number of terms in a join is limited by the number of bits +** in prereqRight and prereqAll. The default is 64 bits, hence SQLite +** is only able to process joins with 64 or fewer tables. +*/ +typedef struct WhereTerm WhereTerm; +struct WhereTerm { + Expr *pExpr; /* Pointer to the subexpression that is this term */ + int iParent; /* Disable pWC->a[iParent] when this term disabled */ + int leftCursor; /* Cursor number of X in "X " */ + union { + int leftColumn; /* Column number of X in "X " */ + WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ + WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + } u; + u16 eOperator; /* A WO_xx value describing */ + u8 wtFlags; /* TERM_xxx bit flags. See below */ + u8 nChild; /* Number of children that must disable us */ + WhereClause *pWC; /* The clause this term is part of */ + Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ + Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ +}; + +/* +** Allowed values of WhereTerm.wtFlags +*/ +#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ +#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ +#define TERM_CODED 0x04 /* This term is already coded */ +#define TERM_COPIED 0x08 /* Has a child */ +#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ +#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ +#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ +#ifdef SQLITE_ENABLE_STAT2 +# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ +#else +# define TERM_VNULL 0x00 /* Disabled if not using stat2 */ +#endif + +/* +** An instance of the following structure holds all information about a +** WHERE clause. Mostly this is a container for one or more WhereTerms. +*/ +struct WhereClause { + Parse *pParse; /* The parser context */ + WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ + Bitmask vmask; /* Bitmask identifying virtual table cursors */ + u8 op; /* Split operator. TK_AND or TK_OR */ + int nTerm; /* Number of terms */ + int nSlot; /* Number of entries in a[] */ + WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ +#if defined(SQLITE_SMALL_STACK) + WhereTerm aStatic[1]; /* Initial static space for a[] */ +#else + WhereTerm aStatic[8]; /* Initial static space for a[] */ +#endif +}; + +/* +** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereOrInfo { + WhereClause wc; /* Decomposition into subterms */ + Bitmask indexable; /* Bitmask of all indexable tables in the clause */ +}; + +/* +** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereAndInfo { + WhereClause wc; /* The subexpression broken out */ +}; + +/* +** An instance of the following structure keeps track of a mapping +** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. +** +** The VDBE cursor numbers are small integers contained in +** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** clause, the cursor numbers might not begin with 0 and they might +** contain gaps in the numbering sequence. But we want to make maximum +** use of the bits in our bitmasks. This structure provides a mapping +** from the sparse cursor numbers into consecutive integers beginning +** with 0. +** +** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask +** corresponds VDBE cursor number B. The A-th bit of a bitmask is 1<3, 5->1, 8->2, 29->0, +** 57->5, 73->4. Or one of 719 other combinations might be used. It +** does not really matter. What is important is that sparse cursor +** numbers all get mapped into bit numbers that begin with 0 and contain +** no gaps. +*/ +struct WhereMaskSet { + int n; /* Number of assigned cursor values */ + int ix[BMS]; /* Cursor assigned to each bit */ +}; + +/* +** A WhereCost object records a lookup strategy and the estimated +** cost of pursuing that strategy. +*/ +struct WhereCost { + WherePlan plan; /* The lookup strategy */ + double rCost; /* Overall cost of pursuing this search strategy */ + Bitmask used; /* Bitmask of cursors used by this plan */ +}; + +/* +** Bitmasks for the operators that indices are able to exploit. An +** OR-ed combination of these values can be used when searching for +** terms in the where clause. +*/ +#define WO_IN 0x001 +#define WO_EQ 0x002 +#define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) +#define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) +#define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) +#define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) +#define WO_MATCH 0x040 +#define WO_ISNULL 0x080 +#define WO_OR 0x100 /* Two or more OR-connected terms */ +#define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_NOOP 0x800 /* This term does not restrict search space */ + +#define WO_ALL 0xfff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ + +/* +** Value for wsFlags returned by bestIndex() and stored in +** WhereLevel.wsFlags. These flags determine which search +** strategies are appropriate. +** +** The least significant 12 bits is reserved as a mask for WO_ values above. +** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL. +** But if the table is the right table of a left join, WhereLevel.wsFlags +** is set to WO_IN|WO_EQ. The WhereLevel.wsFlags field can then be used as +** the "op" parameter to findTerm when we are resolving equality constraints. +** ISNULL constraints will then not be used on the right table of a left +** join. Tickets #2177 and #2189. +*/ +#define WHERE_ROWID_EQ 0x00001000 /* rowid=EXPR or rowid IN (...) */ +#define WHERE_ROWID_RANGE 0x00002000 /* rowidEXPR */ +#define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ +#define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ +#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ +#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ +#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ +#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ +#define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xpParse = pParse; + pWC->pMaskSet = pMaskSet; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; + pWC->vmask = 0; +} + +/* Forward reference */ +static void whereClauseClear(WhereClause*); + +/* +** Deallocate all memory associated with a WhereOrInfo object. +*/ +static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ + whereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Deallocate all memory associated with a WhereAndInfo object. +*/ +static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ + whereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of whereClauseInit(). +*/ +static void whereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); + } +} + +/* +** Add a single new WhereTerm entry to the WhereClause object pWC. +** The new WhereTerm object is constructed from Expr p and with wtFlags. +** The index in pWC->a[] of the new WhereTerm is returned on success. +** 0 is returned if the new WhereTerm could not be added due to a memory +** allocation error. The memory allocation failure will be recorded in +** the db->mallocFailed flag so that higher-level functions can detect it. +** +** This routine will increase the size of the pWC->a[] array as necessary. +** +** If the wtFlags argument includes TERM_DYNAMIC, then responsibility +** for freeing the expression p is assumed by the WhereClause object pWC. +** This is true even if this routine fails to allocate a new WhereTerm. +** +** WARNING: This routine might reallocate the space used to store +** WhereTerms. All pointers to WhereTerms should be invalidated after +** calling this routine. Such pointers may be reinitialized by referencing +** the pWC->a[] array. +*/ +static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ + WhereTerm *pTerm; + int idx; + testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ + if( pWC->nTerm>=pWC->nSlot ){ + WhereTerm *pOld = pWC->a; + sqlite3 *db = pWC->pParse->db; + pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + if( pWC->a==0 ){ + if( wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, p); + } + pWC->a = pOld; + return 0; + } + memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); + if( pOld!=pWC->aStatic ){ + sqlite3DbFree(db, pOld); + } + pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + } + pTerm = &pWC->a[idx = pWC->nTerm++]; + pTerm->pExpr = p; + pTerm->wtFlags = wtFlags; + pTerm->pWC = pWC; + pTerm->iParent = -1; + return idx; +} + +/* +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: +** +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] +** +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. +** +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. +*/ +static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ + pWC->op = (u8)op; + if( pExpr==0 ) return; + if( pExpr->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + whereSplit(pWC, pExpr->pLeft, op); + whereSplit(pWC, pExpr->pRight, op); + } +} + +/* +** Initialize an expression mask set (a WhereMaskSet object) +*/ +#define initMaskSet(P) memset(P, 0, sizeof(*P)) + +/* +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. +*/ +static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ + int i; + assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return ((Bitmask)1)<ix[] +** array will never overflow. +*/ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} + +/* +** This routine walks (recursively) an expression tree and generates +** a bitmask indicating which tables are used in that expression +** tree. +** +** In order for this routine to work, the calling function must have +** previously invoked sqlite3ResolveExprNames() on the expression. See +** the header comment on that routine for additional information. +** The sqlite3ResolveExprNames() routines looks for column names and +** sets their opcodes to TK_COLUMN and their Expr.iTable fields to +** the VDBE cursor number of the table. This routine just has to +** translate the cursor numbers into bitmask values and OR all +** the bitmasks together. +*/ +static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); +static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); +static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask = 0; + if( p==0 ) return 0; + if( p->op==TK_COLUMN ){ + mask = getMask(pMaskSet, p->iTable); + return mask; + } + mask = exprTableUsage(pMaskSet, p->pRight); + mask |= exprTableUsage(pMaskSet, p->pLeft); + if( ExprHasProperty(p, EP_xIsSelect) ){ + mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); + }else{ + mask |= exprListTableUsage(pMaskSet, p->x.pList); + } + return mask; +} +static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); + } + } + return mask; +} +static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + mask |= exprListTableUsage(pMaskSet, pS->pEList); + mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); + mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); + mask |= exprTableUsage(pMaskSet, pS->pWhere); + mask |= exprTableUsage(pMaskSet, pS->pHaving); + pS = pS->pPrior; + } + return mask; +} + +/* +** Return TRUE if the given operator is one of the operators that is +** allowed for an indexable WHERE clause term. The allowed operators are +** "=", "<", ">", "<=", ">=", and "IN". +** +** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be +** of one of the following forms: column = expression column > expression +** column >= expression column < expression column <= expression +** expression = column expression > column expression >= column +** expression < column expression <= column column IN +** (expression-list) column IN (subquery) column IS NULL +*/ +static int allowedOp(int op){ + assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; +} + +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} + +/* +** Commute a comparison operator. Expressions of the form "X op Y" +** are converted into "Y op X". +** +** If a collation sequence is associated with either the left or right +** side of the comparison, it remains associated with the same side after +** the commutation. So "Y collate NOCASE op X" becomes +** "X collate NOCASE op Y". This is because any collation sequence on +** the left hand side of a comparison overrides any collation sequence +** attached to the right. For the same reason the EP_ExpCollate flag +** is not commuted. +*/ +static void exprCommute(Parse *pParse, Expr *pExpr){ + u16 expRight = (pExpr->pRight->flags & EP_ExpCollate); + u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate); + assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); + pExpr->pRight->pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); + pExpr->pLeft->pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl); + pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft; + pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight; + SWAP(Expr*,pExpr->pRight,pExpr->pLeft); + if( pExpr->op>=TK_GT ){ + assert( TK_LT==TK_GT+2 ); + assert( TK_GE==TK_LE+2 ); + assert( TK_GT>TK_EQ ); + assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); + pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; + } +} + +/* +** Translate from TK_xx operator to WO_xx bitmask. +*/ +static u16 operatorMask(int op){ + u16 c; + assert( allowedOp(op) ); + if( op==TK_IN ){ + c = WO_IN; + }else if( op==TK_ISNULL ){ + c = WO_ISNULL; + }else{ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + } + assert( op!=TK_ISNULL || c==WO_ISNULL ); + assert( op!=TK_IN || c==WO_IN ); + assert( op!=TK_EQ || c==WO_EQ ); + assert( op!=TK_LT || c==WO_LT ); + assert( op!=TK_LE || c==WO_LE ); + assert( op!=TK_GT || c==WO_GT ); + assert( op!=TK_GE || c==WO_GE ); + return c; +} + +/* +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur and is one of +** the WO_xx operator codes specified by the op parameter. +** Return a pointer to the term. Return 0 if not found. +*/ +static WhereTerm *findTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +){ + WhereTerm *pTerm; + int k; + assert( iCur>=0 ); + op &= WO_ALL; + for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ + if( pTerm->leftCursor==iCur + && (pTerm->prereqRight & notReady)==0 + && pTerm->u.leftColumn==iColumn + && (pTerm->eOperator & op)!=0 + ){ + if( pIdx && pTerm->eOperator!=WO_ISNULL ){ + Expr *pX = pTerm->pExpr; + CollSeq *pColl; + char idxaff; + int j; + Parse *pParse = pWC->pParse; + + idxaff = pIdx->pTable->aCol[iColumn].affinity; + if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; + + /* Figure out the collation sequence required from an index for + ** it to be useful for optimising expression pX. Store this + ** value in variable pColl. + */ + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + assert(pColl || pParse->nErr); + + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; + } + return pTerm; + } + } + return 0; +} + +/* Forward reference */ +static void exprAnalyze(SrcList*, WhereClause*, int); + +/* +** Call exprAnalyze on all terms in a WHERE clause. +** +** +*/ +static void exprAnalyzeAll( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ +){ + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); + } +} + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION +/* +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. +** +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. +*/ +static int isLikeOrGlob( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* Test this expression */ + Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ + int *pisComplete, /* True if the only wildcard is % in the last character */ + int *pnoCase /* True if uppercase is equivalent to lowercase */ +){ + const char *z = 0; /* String on RHS of LIKE operator */ + Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ + ExprList *pList; /* List of operands to the LIKE operator */ + int c; /* One character in z[] */ + int cnt; /* Number of non-wildcard prefix characters */ + char wc[3]; /* Wildcard characters */ + sqlite3 *db = pParse->db; /* Database connection */ + sqlite3_value *pVal = 0; + int op; /* Opcode of pRight */ + + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + return 0; + } +#ifdef SQLITE_EBCDIC + if( *pnoCase ) return 0; +#endif + pList = pExpr->x.pList; + pLeft = pList->a[1].pExpr; + if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ + /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must + ** be the name of an indexed column with TEXT affinity. */ + return 0; + } + assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ + + pRight = pList->a[0].pExpr; + op = pRight->op; + if( op==TK_REGISTER ){ + op = pRight->op2; + } + if( op==TK_VARIABLE ){ + Vdbe *pReprepare = pParse->pReprepare; + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE); + if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ + z = (char *)sqlite3_value_text(pVal); + } + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ + assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); + }else if( op==TK_STRING ){ + z = pRight->u.zToken; + } + if( z ){ + cnt = 0; + while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ + cnt++; + } + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + Expr *pPrefix; + *pisComplete = c==wc[0] && z[cnt+1]==0; + pPrefix = sqlite3Expr(db, TK_STRING, z); + if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + *ppPrefix = pPrefix; + if( op==TK_VARIABLE ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ + if( *pisComplete && pRight->u.zToken[1] ){ + /* If the rhs of the LIKE expression is a variable, and the current + ** value of the variable means there is no need to invoke the LIKE + ** function, then no OP_Variable will be added to the program. + ** This causes problems for the sqlite3_bind_parameter_name() + ** API. To workaround them, add a dummy OP_Variable here. + */ + int r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCodeTarget(pParse, pRight, r1); + sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); + sqlite3ReleaseTempReg(pParse, r1); + } + } + }else{ + z = 0; + } + } + + sqlite3ValueFree(pVal); + return (z!=0); +} +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Check to see if the given expression is of the form +** +** column MATCH expr +** +** If it is then return TRUE. If not, return FALSE. +*/ +static int isMatchOfColumn( + Expr *pExpr /* Test this expression */ +){ + ExprList *pList; + + if( pExpr->op!=TK_FUNCTION ){ + return 0; + } + if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ + return 0; + } + pList = pExpr->x.pList; + if( pList->nExpr!=2 ){ + return 0; + } + if( pList->a[1].pExpr->op != TK_COLUMN ){ + return 0; + } + return 1; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** If the pBase expression originated in the ON or USING clause of +** a join, then transfer the appropriate markings over to derived. +*/ +static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; +} + +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Analyze a term that consists of two or more OR-connected +** subterms. So in: +** +** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) +** ^^^^^^^^^^^^^^^^^^^^ +** +** This routine analyzes terms such as the middle term in the above example. +** A WhereOrTerm object is computed and attached to the term under +** analysis, regardless of the outcome of the analysis. Hence: +** +** WhereTerm.wtFlags |= TERM_ORINFO +** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object +** +** The term being analyzed must have two or more of OR-connected subterms. +** A single subterm might be a set of AND-connected sub-subterms. +** Examples of terms under analysis: +** +** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 +** (B) x=expr1 OR expr2=x OR x=expr3 +** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) +** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') +** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** +** CASE 1: +** +** If all subterms are of the form T.C=expr for some single column of C +** a single table T (as shown in example B above) then create a new virtual +** term that is an equivalent IN expression. In other words, if the term +** being analyzed is: +** +** x = expr1 OR expr2 = x OR x = expr3 +** +** then create a new virtual term like this: +** +** x IN (expr1,expr2,expr3) +** +** CASE 2: +** +** If all subterms are indexable by a single table T, then set +** +** WhereTerm.eOperator = WO_OR +** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** +** A subterm is "indexable" if it is of the form +** "T.C " where C is any column of table T and +** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". +** A subterm is also indexable if it is an AND of two or more +** subsubterms at least one of which is indexable. Indexable AND +** subterms have their eOperator set to WO_AND and they have +** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** +** From another point of view, "indexable" means that the subterm could +** potentially be used with an index if an appropriate index exists. +** This analysis does not consider whether or not the index exists; that +** is something the bestIndex() routine will determine. This analysis +** only looks at whether subterms appropriate for indexing exist. +** +** All examples A through E above all satisfy case 2. But if a term +** also statisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 2 is not +** satisfied. +** +** It might be the case that multiple tables are indexable. For example, +** (E) above is indexable on tables P, Q, and R. +** +** Terms that satisfy case 2 are candidates for lookup by using +** separate indices to find rowids for each subterm and composing +** the union of all rowids using a RowSet object. This is similar +** to "bitmap indices" in other database engines. +** +** OTHERWISE: +** +** If neither case 1 nor case 2 apply, then leave the eOperator set to +** zero. This term is not useful for search. +*/ +static void exprAnalyzeOrTerm( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the complete WHERE clause */ + int idxTerm /* Index of the OR-term to be analyzed */ +){ + Parse *pParse = pWC->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ + Expr *pExpr = pTerm->pExpr; /* The expression of the term */ + WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */ + int i; /* Loop counters */ + WhereClause *pOrWc; /* Breakup of pTerm into subterms */ + WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ + WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ + Bitmask chngToIN; /* Tables that might satisfy case 1 */ + Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + + /* + ** Break the OR clause into its separate subterms. The subterms are + ** stored in a WhereClause structure containing within the WhereOrInfo + ** object that is attached to the original OR clause term. + */ + assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); + assert( pExpr->op==TK_OR ); + pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); + if( pOrInfo==0 ) return; + pTerm->wtFlags |= TERM_ORINFO; + pOrWc = &pOrInfo->wc; + whereClauseInit(pOrWc, pWC->pParse, pMaskSet); + whereSplit(pOrWc, pExpr, TK_OR); + exprAnalyzeAll(pSrc, pOrWc); + if( db->mallocFailed ) return; + assert( pOrWc->nTerm>=2 ); + + /* + ** Compute the set of tables that might satisfy cases 1 or 2. + */ + indexable = ~(Bitmask)0; + chngToIN = ~(pWC->vmask); + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ + if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ + WhereAndInfo *pAndInfo; + assert( pOrTerm->eOperator==0 ); + assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); + chngToIN = 0; + pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + if( pAndInfo ){ + WhereClause *pAndWC; + WhereTerm *pAndTerm; + int j; + Bitmask b = 0; + pOrTerm->u.pAndInfo = pAndInfo; + pOrTerm->wtFlags |= TERM_ANDINFO; + pOrTerm->eOperator = WO_AND; + pAndWC = &pAndInfo->wc; + whereClauseInit(pAndWC, pWC->pParse, pMaskSet); + whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + exprAnalyzeAll(pSrc, pAndWC); + testcase( db->mallocFailed ); + if( !db->mallocFailed ){ + for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ + assert( pAndTerm->pExpr ); + if( allowedOp(pAndTerm->pExpr->op) ){ + b |= getMask(pMaskSet, pAndTerm->leftCursor); + } + } + } + indexable &= b; + } + }else if( pOrTerm->wtFlags & TERM_COPIED ){ + /* Skip this term for now. We revisit it when we process the + ** corresponding TERM_VIRTUAL term */ + }else{ + Bitmask b; + b = getMask(pMaskSet, pOrTerm->leftCursor); + if( pOrTerm->wtFlags & TERM_VIRTUAL ){ + WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; + b |= getMask(pMaskSet, pOther->leftCursor); + } + indexable &= b; + if( pOrTerm->eOperator!=WO_EQ ){ + chngToIN = 0; + }else{ + chngToIN &= b; + } + } + } + + /* + ** Record the set of tables that satisfy case 2. The set might be + ** empty. + */ + pOrInfo->indexable = indexable; + pTerm->eOperator = indexable==0 ? 0 : WO_OR; + + /* + ** chngToIN holds a set of tables that *might* satisfy case 1. But + ** we have to do some additional checking to see if case 1 really + ** is satisfied. + ** + ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means + ** that there is no possibility of transforming the OR clause into an + ** IN operator because one or more terms in the OR clause contain + ** something other than == on a column in the single table. The 1-bit + ** case means that every term of the OR clause is of the form + ** "table.column=expr" for some single table. The one bit that is set + ** will correspond to the common table. We still need to check to make + ** sure the same column is used on all terms. The 2-bit case is when + ** the all terms are of the form "table1.column=table2.column". It + ** might be possible to form an IN operator with either table1.column + ** or table2.column as the LHS if either is common to every term of + ** the OR clause. + ** + ** Note that terms of the form "table.column1=table.column2" (the + ** same table on both sizes of the ==) cannot be optimized. + */ + if( chngToIN ){ + int okToChngToIN = 0; /* True if the conversion to IN is valid */ + int iColumn = -1; /* Column index on lhs of IN operator */ + int iCursor = -1; /* Table cursor common to all terms */ + int j = 0; /* Loop counter */ + + /* Search for a table and column that appears on one side or the + ** other of the == operator in every subterm. That table and column + ** will be recorded in iCursor and iColumn. There might not be any + ** such table and column. Set okToChngToIN if an appropriate table + ** and column is found but leave okToChngToIN false if not found. + */ + for(j=0; j<2 && !okToChngToIN; j++){ + pOrTerm = pOrWc->a; + for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ + assert( pOrTerm->eOperator==WO_EQ ); + pOrTerm->wtFlags &= ~TERM_OR_OK; + if( pOrTerm->leftCursor==iCursor ){ + /* This is the 2-bit case and we are on the second iteration and + ** current term is from the first iteration. So skip this term. */ + assert( j==1 ); + continue; + } + if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ){ + /* This term must be of the form t1.a==t2.b where t2 is in the + ** chngToIN set but t1 is not. This term will be either preceeded + ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** and use its inversion. */ + testcase( pOrTerm->wtFlags & TERM_COPIED ); + testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); + assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); + continue; + } + iColumn = pOrTerm->u.leftColumn; + iCursor = pOrTerm->leftCursor; + break; + } + if( i<0 ){ + /* No candidate table+column was found. This can only occur + ** on the second iteration */ + assert( j==1 ); + assert( (chngToIN&(chngToIN-1))==0 ); + assert( chngToIN==getMask(pMaskSet, iCursor) ); + break; + } + testcase( j==1 ); + + /* We have found a candidate table and column. Check to see if that + ** table and column is common to every term in the OR clause */ + okToChngToIN = 1; + for(; i>=0 && okToChngToIN; i--, pOrTerm++){ + assert( pOrTerm->eOperator==WO_EQ ); + if( pOrTerm->leftCursor!=iCursor ){ + pOrTerm->wtFlags &= ~TERM_OR_OK; + }else if( pOrTerm->u.leftColumn!=iColumn ){ + okToChngToIN = 0; + }else{ + int affLeft, affRight; + /* If the right-hand side is also a column, then the affinities + ** of both right and left sides must be such that no type + ** conversions are required on the right. (Ticket #2249) + */ + affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); + affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); + if( affRight!=0 && affRight!=affLeft ){ + okToChngToIN = 0; + }else{ + pOrTerm->wtFlags |= TERM_OR_OK; + } + } + } + } + + /* At this point, okToChngToIN is true if original pTerm satisfies + ** case 1. In that case, construct a new virtual term that is + ** pTerm converted into an IN operator. + ** + ** EV: R-00211-15100 + */ + if( okToChngToIN ){ + Expr *pDup; /* A transient duplicate expression */ + ExprList *pList = 0; /* The RHS of the IN operator */ + Expr *pLeft = 0; /* The LHS of the IN operator */ + Expr *pNew; /* The complete IN operator */ + + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ + if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->leftCursor==iCursor ); + assert( pOrTerm->u.leftColumn==iColumn ); + pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); + pList = sqlite3ExprListAppend(pWC->pParse, pList, pDup); + pLeft = pOrTerm->pExpr->pLeft; + } + assert( pLeft!=0 ); + pDup = sqlite3ExprDup(db, pLeft, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); + if( pNew ){ + int idxNew; + transferJoinMarkings(pNew, pExpr); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + pNew->x.pList = pList; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + pWC->a[idxNew].iParent = idxTerm; + pTerm->nChild = 1; + }else{ + sqlite3ExprListDelete(db, pList); + } + pTerm->eOperator = WO_NOOP; /* case 1 trumps case 2 */ + } + } +} +#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ + + +/* +** The input to this routine is an WhereTerm structure with only the +** "pExpr" field filled in. The job of this routine is to analyze the +** subexpression and populate all the other fields of the WhereTerm +** structure. +** +** If the expression is of the form " X" it gets commuted +** to the standard form of "X ". +** +** If the expression is of the form "X Y" where both X and Y are +** columns, then the original expression is unchanged and a new virtual +** term of the form "Y X" is added to the WHERE clause and +** analyzed separately. The original term is marked with TERM_COPIED +** and the new term is marked with TERM_DYNAMIC (because it's pExpr +** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it +** is a commuted copy of a prior term.) The original term has nChild=1 +** and the copy has idxParent set to the index of the original term. +*/ +static void exprAnalyze( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ +){ + WhereTerm *pTerm; /* The term to be analyzed */ + WhereMaskSet *pMaskSet; /* Set of table index masks */ + Expr *pExpr; /* The expression to be analyzed */ + Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ + Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ + int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ + int noCase = 0; /* LIKE/GLOB distinguishes case */ + int op; /* Top-level operator. pExpr->op */ + Parse *pParse = pWC->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection */ + + if( db->mallocFailed ){ + return; + } + pTerm = &pWC->a[idxTerm]; + pMaskSet = pWC->pMaskSet; + pExpr = pTerm->pExpr; + prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); + op = pExpr->op; + if( op==TK_IN ){ + assert( pExpr->pRight==0 ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); + }else{ + pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); + } + }else if( op==TK_ISNULL ){ + pTerm->prereqRight = 0; + }else{ + pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); + } + prereqAll = exprTableUsage(pMaskSet, pExpr); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + } + pTerm->prereqAll = prereqAll; + pTerm->leftCursor = -1; + pTerm->iParent = -1; + pTerm->eOperator = 0; + if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + if( pLeft->op==TK_COLUMN ){ + pTerm->leftCursor = pLeft->iTable; + pTerm->u.leftColumn = pLeft->iColumn; + pTerm->eOperator = operatorMask(op); + } + if( pRight && pRight->op==TK_COLUMN ){ + WhereTerm *pNew; + Expr *pDup; + if( pTerm->leftCursor>=0 ){ + int idxNew; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew==0 ) return; + pNew = &pWC->a[idxNew]; + pNew->iParent = idxTerm; + pTerm = &pWC->a[idxTerm]; + pTerm->nChild = 1; + pTerm->wtFlags |= TERM_COPIED; + }else{ + pDup = pExpr; + pNew = pTerm; + } + exprCommute(pParse, pDup); + pLeft = pDup->pLeft; + pNew->leftCursor = pLeft->iTable; + pNew->u.leftColumn = pLeft->iColumn; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; + pNew->prereqAll = prereqAll; + pNew->eOperator = operatorMask(pDup->op); + } + } + +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION + /* If a term is the BETWEEN operator, create two new virtual terms + ** that define the range that the BETWEEN implements. For example: + ** + ** a BETWEEN b AND c + ** + ** is converted into: + ** + ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) + ** + ** The two new terms are added onto the end of the WhereClause object. + ** The new terms are "dynamic" and are children of the original BETWEEN + ** term. That means that if the BETWEEN term is coded, the children are + ** skipped. Or, if the children are satisfied by an index, the original + ** BETWEEN term is skipped. + */ + else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ + ExprList *pList = pExpr->x.pList; + int i; + static const u8 ops[] = {TK_GE, TK_LE}; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + for(i=0; i<2; i++){ + Expr *pNewExpr; + int idxNew; + pNewExpr = sqlite3PExpr(pParse, ops[i], + sqlite3ExprDup(db, pExpr->pLeft, 0), + sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + pWC->a[idxNew].iParent = idxTerm; + } + pTerm->nChild = 2; + } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ + +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) + /* Analyze a term that is composed of two or more subterms connected by + ** an OR operator. + */ + else if( pExpr->op==TK_OR ){ + assert( pWC->op==TK_AND ); + exprAnalyzeOrTerm(pSrc, pWC, idxTerm); + pTerm = &pWC->a[idxTerm]; + } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + ** + ** A like pattern of the form "x LIKE 'abc%'" is changed into constraints + ** + ** x>='abc' AND x<'abd' AND x LIKE 'abc%' + ** + ** The last character of the prefix "abc" is incremented to form the + ** termination condition "abd". + */ + if( pWC->op==TK_AND + && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) + ){ + Expr *pLeft; /* LHS of LIKE/GLOB operator */ + Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ + Expr *pNewExpr1; + Expr *pNewExpr2; + int idxNew1; + int idxNew2; + CollSeq *pColl; /* Collating sequence to use */ + + pLeft = pExpr->x.pList->a[1].pExpr; + pStr2 = sqlite3ExprDup(db, pStr1, 0); + if( !db->mallocFailed ){ + u8 c, *pC; /* Last character before the first wildcard */ + pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; + c = *pC; + if( noCase ){ + /* The point is to increment the last character before the first + ** wildcard. But if we increment '@', that will push it into the + ** alphabetic range where case conversions will mess up the + ** inequality. To avoid this, make sure to also run the full + ** LIKE on all candidate expressions by clearing the isComplete flag + */ + if( c=='A'-1 ) isComplete = 0; /* EV: R-64339-08207 */ + + + c = sqlite3UpperToLower[c]; + } + *pC = c + 1; + } + pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr1, 0); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew1==0 ); + exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr2, 0); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew2); + pTerm = &pWC->a[idxTerm]; + if( isComplete ){ + pWC->a[idxNew1].iParent = idxTerm; + pWC->a[idxNew2].iParent = idxTerm; + pTerm->nChild = 2; + } + } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Add a WO_MATCH auxiliary term to the constraint set if the + ** current expression is of the form: column MATCH expr. + ** This information is used by the xBestIndex methods of + ** virtual tables. The native query optimizer does not attempt + ** to do anything with MATCH functions. + */ + if( isMatchOfColumn(pExpr) ){ + int idxNew; + Expr *pRight, *pLeft; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; + + pRight = pExpr->x.pList->a[0].pExpr; + pLeft = pExpr->x.pList->a[1].pExpr; + prereqExpr = exprTableUsage(pMaskSet, pRight); + prereqColumn = exprTableUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0), 0); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_MATCH; + pNewTerm->iParent = idxTerm; + pTerm = &pWC->a[idxTerm]; + pTerm->nChild = 1; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef SQLITE_ENABLE_STAT2 + /* When sqlite_stat2 histogram data is available an operator of the + ** form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** Note that the virtual term must be tagged with TERM_VNULL. This + ** TERM_VNULL tag will suppress the not-null check at the beginning + ** of the loop. Without the TERM_VNULL flag, the not-null check at + ** the start of the loop will prevent any results from being returned. + */ + if( pExpr->op==TK_NOTNULL + && pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; + + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); + + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + pNewTerm->iParent = idxTerm; + pTerm = &pWC->a[idxTerm]; + pTerm->nChild = 1; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } +#endif /* SQLITE_ENABLE_STAT2 */ + + /* Prevent ON clause terms of a LEFT JOIN from being used to drive + ** an index for tables to the left of the join. + */ + pTerm->prereqRight |= extraRight; +} + +/* +** Return TRUE if any of the expressions in pList->a[iFirst...] contain +** a reference to any table other than the iBase table. +*/ +static int referencesOtherTables( + ExprList *pList, /* Search expressions in ths list */ + WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ + int iFirst, /* Be searching with the iFirst-th expression */ + int iBase /* Ignore references to this table */ +){ + Bitmask allowed = ~getMask(pMaskSet, iBase); + while( iFirstnExpr ){ + if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ + return 1; + } + } + return 0; +} + + +/* +** This routine decides if pIdx can be used to satisfy the ORDER BY +** clause. If it can, it returns 1. If pIdx cannot satisfy the +** ORDER BY clause, this routine returns 0. +** +** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the +** left-most table in the FROM clause of that same SELECT statement and +** the table has a cursor number of "base". pIdx is an index on pTab. +** +** nEqCol is the number of columns of pIdx that are used as equality +** constraints. Any of these columns may be missing from the ORDER BY +** clause and the match can still be a success. +** +** All terms of the ORDER BY that match against the index must be either +** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE +** index do not need to satisfy this constraint.) The *pbRev value is +** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if +** the ORDER BY clause is all ASC. +*/ +static int isSortingIndex( + Parse *pParse, /* Parsing context */ + WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ + Index *pIdx, /* The index we are testing */ + int base, /* Cursor number for the table to be sorted */ + ExprList *pOrderBy, /* The ORDER BY clause */ + int nEqCol, /* Number of index columns with == constraints */ + int wsFlags, /* Index usages flags */ + int *pbRev /* Set to 1 if ORDER BY is DESC */ +){ + int i, j; /* Loop counters */ + int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ + int nTerm; /* Number of ORDER BY terms */ + struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ + sqlite3 *db = pParse->db; + + assert( pOrderBy!=0 ); + nTerm = pOrderBy->nExpr; + assert( nTerm>0 ); + + /* Argument pIdx must either point to a 'real' named index structure, + ** or an index structure allocated on the stack by bestBtreeIndex() to + ** represent the rowid index that is part of every table. */ + assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); + + /* Match terms of the ORDER BY clause against columns of + ** the index. + ** + ** Note that indices have pIdx->nColumn regular columns plus + ** one additional column containing the rowid. The rowid column + ** of the index is also allowed to match against the ORDER BY + ** clause. + */ + for(i=j=0, pTerm=pOrderBy->a; jnColumn; i++){ + Expr *pExpr; /* The expression of the ORDER BY pTerm */ + CollSeq *pColl; /* The collating sequence of pExpr */ + int termSortOrder; /* Sort order for this term */ + int iColumn; /* The i-th column of the index. -1 for rowid */ + int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + const char *zColl; /* Name of the collating sequence for i-th index term */ + + pExpr = pTerm->pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ + /* Can not use an index sort on anything that is not a column in the + ** left-most table of the FROM clause */ + break; + } + pColl = sqlite3ExprCollSeq(pParse, pExpr); + if( !pColl ){ + pColl = db->pDfltColl; + } + if( pIdx->zName && inColumn ){ + iColumn = pIdx->aiColumn[i]; + if( iColumn==pIdx->pTable->iPKey ){ + iColumn = -1; + } + iSortOrder = pIdx->aSortOrder[i]; + zColl = pIdx->azColl[i]; + }else{ + iColumn = -1; + iSortOrder = 0; + zColl = pColl->zName; + } + if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ + /* Term j of the ORDER BY clause does not match column i of the index */ + if( inColumn ){ + /* Index column i is the rowid. All other terms match. */ + break; + }else{ + /* If an index column fails to match and is not constrained by == + ** then the index cannot satisfy the ORDER BY constraint. + */ + return 0; + } + } + assert( pIdx->aSortOrder!=0 || iColumn==-1 ); + assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + termSortOrder = iSortOrder ^ pTerm->sortOrder; + if( i>nEqCol ){ + if( termSortOrder!=sortOrder ){ + /* Indices can only be used if all ORDER BY terms past the + ** equality constraints are all either DESC or ASC. */ + return 0; + } + }else{ + sortOrder = termSortOrder; + } + j++; + pTerm++; + if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ + /* If the indexed column is the primary key and everything matches + ** so far and none of the ORDER BY terms to the right reference other + ** tables in the join, then we are assured that the index can be used + ** to sort because the primary key is unique and so none of the other + ** columns will make any difference + */ + j = nTerm; + } + } + + *pbRev = sortOrder!=0; + if( j>=nTerm ){ + /* All terms of the ORDER BY clause are covered by this index so + ** this index can be used for sorting. */ + return 1; + } + if( pIdx->onError!=OE_None && i==pIdx->nColumn + && (wsFlags & WHERE_COLUMN_NULL)==0 + && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ + /* All terms of this index match some prefix of the ORDER BY clause + ** and the index is UNIQUE and no terms on the tail of the ORDER BY + ** clause reference other tables in a join. If this is all true then + ** the order by clause is superfluous. Not that if the matching + ** condition is IS NULL then the result is not necessarily unique + ** even on a UNIQUE index, so disallow those cases. */ + return 1; + } + return 0; +} + +/* +** Prepare a crude estimate of the logarithm of the input value. +** The results need not be exact. This is only used for estimating +** the total cost of performing operations with O(logN) or O(NlogN) +** complexity. Because N is just a guess, it is no great tragedy if +** logN is a little off. +*/ +static double estLog(double N){ + double logN = 1; + double x = 10; + while( N>x ){ + logN += 1; + x *= 10; + } + return logN; +} + +/* +** Two routines for printing the content of an sqlite3_index_info +** structure. Used for testing and debugging only. If neither +** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines +** are no-ops. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_DEBUG) +static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable); + } + for(i=0; inOrderBy; i++){ + sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } +} +static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", + i, + p->aConstraintUsage[i].argvIndex, + p->aConstraintUsage[i].omit); + } + sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); + sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); + sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); + sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); +} +#else +#define TRACE_IDX_INPUTS(A) +#define TRACE_IDX_OUTPUTS(A) +#endif + +/* +** Required because bestIndex() is called by bestOrClauseIndex() +*/ +static void bestIndex( + Parse*, WhereClause*, struct SrcList_item*, + Bitmask, Bitmask, ExprList*, WhereCost*); + +/* +** This routine attempts to find an scanning strategy that can be used +** to optimize an 'OR' expression that is part of a WHERE clause. +** +** The table associated with FROM clause term pSrc may be either a +** regular B-Tree table or a virtual table. +*/ +static void bestOrClauseIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to search */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ + ExprList *pOrderBy, /* The ORDER BY clause */ + WhereCost *pCost /* Lowest cost query plan */ +){ +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ + const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ + WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + + /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses + ** are used */ + if( pSrc->notIndexed || pSrc->pIndex!=0 ){ + return; + } + + /* Search the WHERE clause terms for a usable WO_OR term. */ + for(pTerm=pWC->a; pTermeOperator==WO_OR + && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 + && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + int flags = WHERE_MULTI_OR; + double rTotal = 0; + double nRow = 0; + Bitmask used = 0; + + for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) + )); + if( pOrTerm->eOperator==WO_AND ){ + WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; + bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); + }else if( pOrTerm->leftCursor==iCur ){ + WhereClause tempWC; + tempWC.pParse = pWC->pParse; + tempWC.pMaskSet = pWC->pMaskSet; + tempWC.op = TK_AND; + tempWC.a = pOrTerm; + tempWC.nTerm = 1; + bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); + }else{ + continue; + } + rTotal += sTermCost.rCost; + nRow += sTermCost.plan.nRow; + used |= sTermCost.used; + if( rTotal>=pCost->rCost ) break; + } + + /* If there is an ORDER BY clause, increase the scan cost to account + ** for the cost of the sort. */ + if( pOrderBy!=0 ){ + WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", + rTotal, rTotal+nRow*estLog(nRow))); + rTotal += nRow*estLog(nRow); + } + + /* If the cost of scanning using this OR term for optimization is + ** less than the current cost stored in pCost, replace the contents + ** of pCost. */ + WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); + if( rTotalrCost ){ + pCost->rCost = rTotal; + pCost->used = used; + pCost->plan.nRow = nRow; + pCost->plan.wsFlags = flags; + pCost->plan.u.pTerm = pTerm; + } + } + } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ +} + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return TRUE if the WHERE clause term pTerm is of a form where it +** could be used with an index to access pSrc, assuming an appropriate +** index existed. +*/ +static int termCanDriveIndex( + WhereTerm *pTerm, /* WHERE clause term to check */ + struct SrcList_item *pSrc, /* Table we are trying to access */ + Bitmask notReady /* Tables in outer loops of the join */ +){ + char aff; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; + if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->prereqRight & notReady)!=0 ) return 0; + aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; + if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + return 1; +} +#endif + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** If the query plan for pSrc specified in pCost is a full table scan +** and indexing is allows (if there is no NOT INDEXED clause) and it +** possible to construct a transient index that would perform better +** than a full table scan even when the cost of constructing the index +** is taken into account, then alter the query plan to use the +** transient index. +*/ +static void bestAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to search */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereCost *pCost /* Lowest cost query plan */ +){ + double nTableRow; /* Rows in the input table */ + double logN; /* log(nTableRow) */ + double costTempIdx; /* per-query cost of the transient index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + Table *pTable; /* Table tht might be indexed */ + + if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ + /* Automatic indices are disabled at run-time */ + return; + } + if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ + /* We already have some kind of index in use for this query. */ + return; + } + if( pSrc->notIndexed ){ + /* The NOT INDEXED clause appears in the SQL. */ + return; + } + + assert( pParse->nQueryLoop >= (double)1 ); + pTable = pSrc->pTab; + nTableRow = pTable->nRowEst; + logN = estLog(nTableRow); + costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); + if( costTempIdx>=pCost->rCost ){ + /* The cost of creating the transient table would be greater than + ** doing the full table scan */ + return; + } + + /* Search for any equality comparison term */ + pWCEnd = &pWC->a[pWC->nTerm]; + for(pTerm=pWC->a; pTermrCost, costTempIdx)); + pCost->rCost = costTempIdx; + pCost->plan.nRow = logN + 1; + pCost->plan.wsFlags = WHERE_TEMP_INDEX; + pCost->used = pTerm->prereqRight; + break; + } + } +} +#else +# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Generate code to construct the Index object for an automatic index +** and to set up the WhereLevel object pLevel so that the code generator +** makes use of the automatic index. +*/ +static void constructAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereLevel *pLevel /* Write new index here */ +){ + int nColumn; /* Number of columns in the constructed index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + int nByte; /* Byte of memory needed for pIdx */ + Index *pIdx; /* Object describing the transient index */ + Vdbe *v; /* Prepared statement under construction */ + int regIsInit; /* Register set by initialization */ + int addrInit; /* Address of the initialization bypass jump */ + Table *pTable; /* The table being indexed */ + KeyInfo *pKeyinfo; /* Key information for the index */ + int addrTop; /* Top of the index fill loop */ + int regRecord; /* Register holding an index record */ + int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ + CollSeq *pColl; /* Collating sequence to on a column */ + Bitmask idxCols; /* Bitmap of columns used for indexing */ + Bitmask extraCols; /* Bitmap of additional columns */ + + /* Generate code to skip over the creation and initialization of the + ** transient index on 2nd and subsequent iterations of the loop. */ + v = pParse->pVdbe; + assert( v!=0 ); + regIsInit = ++pParse->nMem; + addrInit = sqlite3VdbeAddOp1(v, OP_If, regIsInit); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regIsInit); + + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nColumn = 0; + pTable = pSrc->pTab; + pWCEnd = &pWC->a[pWC->nTerm]; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<0 ); + pLevel->plan.nEq = nColumn; + + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | (((Bitmask)1)<<(BMS-1))); + mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & (((Bitmask)1)<<(BMS-1)) ){ + nColumn += pTable->nCol - BMS + 1; + } + pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ; + + /* Construct the Index object to describe this index */ + nByte = sizeof(Index); + nByte += nColumn*sizeof(int); /* Index.aiColumn */ + nByte += nColumn*sizeof(char*); /* Index.azColl */ + nByte += nColumn; /* Index.aSortOrder */ + pIdx = sqlite3DbMallocZero(pParse->db, nByte); + if( pIdx==0 ) return; + pLevel->plan.u.pIdx = pIdx; + pIdx->azColl = (char**)&pIdx[1]; + pIdx->aiColumn = (int*)&pIdx->azColl[nColumn]; + pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nColumn]; + pIdx->zName = "auto-index"; + pIdx->nColumn = nColumn; + pIdx->pTable = pTable; + n = 0; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<pExpr; + idxCols |= cMask; + pIdx->aiColumn[n] = pTerm->u.leftColumn; + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pIdx->azColl[n] = ALWAYS(pColl) ? pColl->zName : "BINARY"; + n++; + } + } + } + assert( (u32)n==pLevel->plan.nEq ); + + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + if( pSrc->colUsed & (((Bitmask)1)<<(BMS-1)) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + assert( n==nColumn ); + + /* Create the automatic index */ + pKeyinfo = sqlite3IndexKeyinfo(pParse, pIdx); + assert( pLevel->iIdxCur>=0 ); + sqlite3VdbeAddOp4(v, OP_OpenAutoindex, pLevel->iIdxCur, nColumn+1, 0, + (char*)pKeyinfo, P4_KEYINFO_HANDOFF); + VdbeComment((v, "for %s", pTable->zName)); + + /* Fill the automatic index with content */ + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); + regRecord = sqlite3GetTempReg(pParse); + sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + sqlite3VdbeJumpHere(v, addrTop); + sqlite3ReleaseTempReg(pParse, regRecord); + + /* Jump here when skipping the initialization */ + sqlite3VdbeJumpHere(v, addrInit); +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Allocate and populate an sqlite3_index_info structure. It is the +** responsibility of the caller to eventually release the structure +** by passing the pointer returned by this function to sqlite3_free(). +*/ +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, + WhereClause *pWC, + struct SrcList_item *pSrc, + ExprList *pOrderBy +){ + int i, j; + int nTerm; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_orderby *pIdxOrderBy; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int nOrderBy; + sqlite3_index_info *pIdxInfo; + + WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName)); + + /* Count the number of possible WHERE clause constraints referring + ** to this virtual table */ + for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); + testcase( pTerm->eOperator==WO_IN ); + testcase( pTerm->eOperator==WO_ISNULL ); + if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + nTerm++; + } + + /* If the ORDER BY clause contains only columns in the current + ** virtual table then allocate space for the aOrderBy part of + ** the sqlite3_index_info structure. + */ + nOrderBy = 0; + if( pOrderBy ){ + for(i=0; inExpr; i++){ + Expr *pExpr = pOrderBy->a[i].pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + } + if( i==pOrderBy->nExpr ){ + nOrderBy = pOrderBy->nExpr; + } + } + + /* Allocate the sqlite3_index_info structure + */ + pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + + sizeof(*pIdxOrderBy)*nOrderBy ); + if( pIdxInfo==0 ){ + sqlite3ErrorMsg(pParse, "out of memory"); + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + return 0; + } + + /* Initialize the structure. The sqlite3_index_info structure contains + ** many fields that are declared "const" to prevent xBestIndex from + ** changing them. We have to do some funky casting in order to + ** initialize those fields. + */ + pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; + pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; + pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; + *(int*)&pIdxInfo->nConstraint = nTerm; + *(int*)&pIdxInfo->nOrderBy = nOrderBy; + *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; + *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; + *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = + pUsage; + + for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); + testcase( pTerm->eOperator==WO_IN ); + testcase( pTerm->eOperator==WO_ISNULL ); + if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + pIdxCons[j].iColumn = pTerm->u.leftColumn; + pIdxCons[j].iTermOffset = i; + pIdxCons[j].op = (u8)pTerm->eOperator; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); + assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + j++; + } + for(i=0; ia[i].pExpr; + pIdxOrderBy[i].iColumn = pExpr->iColumn; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + } + + return pIdxInfo; +} + +/* +** The table object reference passed as the second argument to this function +** must represent a virtual table. This function invokes the xBestIndex() +** method of the virtual table with the sqlite3_index_info pointer passed +** as the argument. +** +** If an error occurs, pParse is populated with an error message and a +** non-zero value is returned. Otherwise, 0 is returned and the output +** part of the sqlite3_index_info structure is left populated. +** +** Whether or not an error is returned, it is the responsibility of the +** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates +** that this is required. +*/ +static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ + sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + int i; + int rc; + + WHERETRACE(("xBestIndex for %s\n", pTab->zName)); + TRACE_IDX_INPUTS(p); + rc = pVtab->pModule->xBestIndex(pVtab, p); + TRACE_IDX_OUTPUTS(p); + + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + pParse->db->mallocFailed = 1; + }else if( !pVtab->zErrMsg ){ + sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + }else{ + sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); + } + } + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + + for(i=0; inConstraint; i++){ + if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ + sqlite3ErrorMsg(pParse, + "table %s: xBestIndex returned an invalid plan", pTab->zName); + } + } + + return pParse->nErr; +} + + +/* +** Compute the best index for a virtual table. +** +** The best index is computed by the xBestIndex method of the virtual +** table module. This routine is really just a wrapper that sets up +** the sqlite3_index_info structure that is used to communicate with +** xBestIndex. +** +** In a join, this routine might be called multiple times for the +** same virtual table. The sqlite3_index_info structure is created +** and initialized on the first invocation and reused on all subsequent +** invocations. The sqlite3_index_info structure is also used when +** code is generated to access the virtual table. The whereInfoDelete() +** routine takes care of freeing the sqlite3_index_info structure after +** everybody has finished with it. +*/ +static void bestVirtualIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to search */ + Bitmask notReady, /* Mask of cursors not available for index */ + Bitmask notValid, /* Cursors not valid for any purpose */ + ExprList *pOrderBy, /* The order by clause */ + WhereCost *pCost, /* Lowest cost query plan */ + sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ +){ + Table *pTab = pSrc->pTab; + sqlite3_index_info *pIdxInfo; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int i, j; + int nOrderBy; + double rCost; + + /* Make sure wsFlags is initialized to some sane value. Otherwise, if the + ** malloc in allocateIndexInfo() fails and this function returns leaving + ** wsFlags in an uninitialized state, the caller may behave unpredictably. + */ + memset(pCost, 0, sizeof(*pCost)); + pCost->plan.wsFlags = WHERE_VIRTUALTABLE; + + /* If the sqlite3_index_info structure has not been previously + ** allocated and initialized, then allocate and initialize it now. + */ + pIdxInfo = *ppIdxInfo; + if( pIdxInfo==0 ){ + *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); + } + if( pIdxInfo==0 ){ + return; + } + + /* At this point, the sqlite3_index_info structure that pIdxInfo points + ** to will have been initialized, either during the current invocation or + ** during some prior invocation. Now we just have to customize the + ** details of pIdxInfo for the current invocation and pass it to + ** xBestIndex. + */ + + /* The module name must be defined. Also, by this point there must + ** be a pointer to an sqlite3_vtab structure. Otherwise + ** sqlite3ViewGetColumnNames() would have picked up the error. + */ + assert( pTab->azModuleArg && pTab->azModuleArg[0] ); + assert( sqlite3GetVTable(pParse->db, pTab) ); + + /* Set the aConstraint[].usable fields and initialize all + ** output variables to zero. + ** + ** aConstraint[].usable is true for constraints where the right-hand + ** side contains only references to tables to the left of the current + ** table. In other words, if the constraint is of the form: + ** + ** column = expr + ** + ** and we are evaluating a join, then the constraint on column is + ** only valid if all tables referenced in expr occur to the left + ** of the table containing column. + ** + ** The aConstraints[] array contains entries for all constraints + ** on the current table. That way we only have to compute it once + ** even though we might try to pick the best index multiple times. + ** For each attempt at picking an index, the order of tables in the + ** join might be different so we have to recompute the usable flag + ** each time. + */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + pUsage = pIdxInfo->aConstraintUsage; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + } + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */ + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2); + nOrderBy = pIdxInfo->nOrderBy; + if( !pOrderBy ){ + pIdxInfo->nOrderBy = 0; + } + + if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ + return; + } + + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++){ + if( pUsage[i].argvIndex>0 ){ + pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; + } + } + + /* If there is an ORDER BY clause, and the selected virtual table index + ** does not satisfy it, increase the cost of the scan accordingly. This + ** matches the processing for non-virtual tables in bestBtreeIndex(). + */ + rCost = pIdxInfo->estimatedCost; + if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ + rCost += estLog(rCost)*rCost; + } + + /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the + ** inital value of lowestCost in this loop. If it is, then the + ** (costrCost = (SQLITE_BIG_DBL/((double)2)); + }else{ + pCost->rCost = rCost; + } + pCost->plan.u.pVtabIdx = pIdxInfo; + if( pIdxInfo->orderByConsumed ){ + pCost->plan.wsFlags |= WHERE_ORDERBY; + } + pCost->plan.nEq = 0; + pIdxInfo->nOrderBy = nOrderBy; + + /* Try to find a more efficient access pattern by using multiple indexes + ** to optimize an OR expression within the WHERE clause. + */ + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Argument pIdx is a pointer to an index structure that has an array of +** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column +** stored in Index.aSample. These samples divide the domain of values stored +** the index into (SQLITE_INDEX_SAMPLES+1) regions. +** Region 0 contains all values less than the first sample value. Region +** 1 contains values between the first and second samples. Region 2 contains +** values between samples 2 and 3. And so on. Region SQLITE_INDEX_SAMPLES +** contains values larger than the last sample. +** +** If the index contains many duplicates of a single value, then it is +** possible that two or more adjacent samples can hold the same value. +** When that is the case, the smallest possible region code is returned +** when roundUp is false and the largest possible region code is returned +** when roundUp is true. +** +** If successful, this function determines which of the regions value +** pVal lies in, sets *piRegion to the region index (a value between 0 +** and SQLITE_INDEX_SAMPLES+1, inclusive) and returns SQLITE_OK. +** Or, if an OOM occurs while converting text values between encodings, +** SQLITE_NOMEM is returned and *piRegion is undefined. +*/ +#ifdef SQLITE_ENABLE_STAT2 +static int whereRangeRegion( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + sqlite3_value *pVal, /* Value to consider */ + int roundUp, /* Return largest valid region if true */ + int *piRegion /* OUT: Region of domain in which value lies */ +){ + assert( roundUp==0 || roundUp==1 ); + if( ALWAYS(pVal) ){ + IndexSample *aSample = pIdx->aSample; + int i = 0; + int eType = sqlite3_value_type(pVal); + + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + double r = sqlite3_value_double(pVal); + for(i=0; i=SQLITE_TEXT ) break; + if( roundUp ){ + if( aSample[i].u.r>r ) break; + }else{ + if( aSample[i].u.r>=r ) break; + } + } + }else if( eType==SQLITE_NULL ){ + i = 0; + if( roundUp ){ + while( idb; + CollSeq *pColl; + const u8 *z; + int n; + + /* pVal comes from sqlite3ValueFromExpr() so the type cannot be NULL */ + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + + if( eType==SQLITE_BLOB ){ + z = (const u8 *)sqlite3_value_blob(pVal); + pColl = db->pDfltColl; + assert( pColl->enc==SQLITE_UTF8 ); + }else{ + pColl = sqlite3GetCollSeq(db, SQLITE_UTF8, 0, *pIdx->azColl); + if( pColl==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", + *pIdx->azColl); + return SQLITE_ERROR; + } + z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); + if( !z ){ + return SQLITE_NOMEM; + } + assert( z && pColl && pColl->xCmp ); + } + n = sqlite3ValueBytes(pVal, pColl->enc); + + for(i=0; ienc!=SQLITE_UTF8 ){ + int nSample; + char *zSample = sqlite3Utf8to16( + db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample + ); + if( !zSample ){ + assert( db->mallocFailed ); + return SQLITE_NOMEM; + } + c = pColl->xCmp(pColl->pUser, nSample, zSample, n, z); + sqlite3DbFree(db, zSample); + }else +#endif + { + c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); + } + if( c-roundUp>=0 ) break; + } + } + + assert( i>=0 && i<=SQLITE_INDEX_SAMPLES ); + *piRegion = i; + } + return SQLITE_OK; +} +#endif /* #ifdef SQLITE_ENABLE_STAT2 */ + +/* +** If expression pExpr represents a literal value, set *pp to point to +** an sqlite3_value structure containing the same value, with affinity +** aff applied to it, before returning. It is the responsibility of the +** caller to eventually release this structure by passing it to +** sqlite3ValueFree(). +** +** If the current parse is a recompile (sqlite3Reprepare()) and pExpr +** is an SQL variable that currently has a non-NULL value bound to it, +** create an sqlite3_value structure containing this value, again with +** affinity aff applied to it, instead. +** +** If neither of the above apply, set *pp to NULL. +** +** If an error occurs, return an error code. Otherwise, SQLITE_OK. +*/ +#ifdef SQLITE_ENABLE_STAT2 +static int valueFromExpr( + Parse *pParse, + Expr *pExpr, + u8 aff, + sqlite3_value **pp +){ + if( pExpr->op==TK_VARIABLE + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) + ){ + int iVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ + *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); + return SQLITE_OK; + } + return sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, aff, pp); +} +#endif + +/* +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper +** +** If either of the upper or lower bound is not present, then NULL is passed in +** place of the corresponding WhereTerm. +** +** The nEq parameter is passed the index of the index column subject to the +** range constraint. Or, equivalently, the number of equality constraints +** optimized by the proposed index scan. For example, assuming index p is +** on t1(a, b), and the SQL query is: +** +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... +** +** then nEq should be passed the value 1 (as the range restricted column, +** b, is the second left-most column of the index). Or, if the query is: +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** +** then nEq should be passed 0. +** +** The returned value is an integer between 1 and 100, inclusive. A return +** value of 1 indicates that the proposed range scan is expected to visit +** approximately 1/100th (1%) of the rows selected by the nEq equality +** constraints (if any). A return value of 100 indicates that it is expected +** that the range scan will visit every row (100%) selected by the equality +** constraints. +** +** In the absence of sqlite_stat2 ANALYZE data, each range inequality +** reduces the search space by 3/4ths. Hence a single constraint (x>?) +** results in a return of 25 and a range constraint (x>? AND xaCol[] of the range-compared column */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + int *piEst /* OUT: Return value */ +){ + int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_STAT2 + + if( nEq==0 && p->aSample ){ + sqlite3_value *pLowerVal = 0; + sqlite3_value *pUpperVal = 0; + int iEst; + int iLower = 0; + int iUpper = SQLITE_INDEX_SAMPLES; + int roundUpUpper = 0; + int roundUpLower = 0; + u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; + + if( pLower ){ + Expr *pExpr = pLower->pExpr->pRight; + rc = valueFromExpr(pParse, pExpr, aff, &pLowerVal); + assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); + roundUpLower = (pLower->eOperator==WO_GT) ?1:0; + } + if( rc==SQLITE_OK && pUpper ){ + Expr *pExpr = pUpper->pExpr->pRight; + rc = valueFromExpr(pParse, pExpr, aff, &pUpperVal); + assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); + roundUpUpper = (pUpper->eOperator==WO_LE) ?1:0; + } + + if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){ + sqlite3ValueFree(pLowerVal); + sqlite3ValueFree(pUpperVal); + goto range_est_fallback; + }else if( pLowerVal==0 ){ + rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper); + if( pLower ) iLower = iUpper/2; + }else if( pUpperVal==0 ){ + rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower); + if( pUpper ) iUpper = (iLower + SQLITE_INDEX_SAMPLES + 1)/2; + }else{ + rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper); + if( rc==SQLITE_OK ){ + rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower); + } + } + WHERETRACE(("range scan regions: %d..%d\n", iLower, iUpper)); + + iEst = iUpper - iLower; + testcase( iEst==SQLITE_INDEX_SAMPLES ); + assert( iEst<=SQLITE_INDEX_SAMPLES ); + if( iEst<1 ){ + *piEst = 50/SQLITE_INDEX_SAMPLES; + }else{ + *piEst = (iEst*100)/SQLITE_INDEX_SAMPLES; + } + sqlite3ValueFree(pLowerVal); + sqlite3ValueFree(pUpperVal); + return rc; + } +range_est_fallback: +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(nEq); +#endif + assert( pLower || pUpper ); + *piEst = 100; + if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *piEst /= 4; + if( pUpper ) *piEst /= 4; + return rc; +} + +#ifdef SQLITE_ENABLE_STAT2 +/* +** Estimate the number of rows that will be returned based on +** an equality constraint x=VALUE and where that VALUE occurs in +** the histogram data. This only works when x is the left-most +** column of an index and sqlite_stat2 histogram data is available +** for that index. When pExpr==NULL that means the constraint is +** "x IS NULL" instead of "x=VALUE". +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereEqualScanEst( + Parse *pParse, /* Parsing & code generating context */ + Index *p, /* The index whose left-most column is pTerm */ + Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ + double *pnRow /* Write the revised row estimate here */ +){ + sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ + int iLower, iUpper; /* Range of histogram regions containing pRhs */ + u8 aff; /* Column affinity */ + int rc; /* Subfunction return code */ + double nRowEst; /* New estimate of the number of rows */ + + assert( p->aSample!=0 ); + aff = p->pTable->aCol[p->aiColumn[0]].affinity; + if( pExpr ){ + rc = valueFromExpr(pParse, pExpr, aff, &pRhs); + if( rc ) goto whereEqualScanEst_cancel; + }else{ + pRhs = sqlite3ValueNew(pParse->db); + } + if( pRhs==0 ) return SQLITE_NOTFOUND; + rc = whereRangeRegion(pParse, p, pRhs, 0, &iLower); + if( rc ) goto whereEqualScanEst_cancel; + rc = whereRangeRegion(pParse, p, pRhs, 1, &iUpper); + if( rc ) goto whereEqualScanEst_cancel; + WHERETRACE(("equality scan regions: %d..%d\n", iLower, iUpper)); + if( iLower>=iUpper ){ + nRowEst = p->aiRowEst[0]/(SQLITE_INDEX_SAMPLES*2); + if( nRowEst<*pnRow ) *pnRow = nRowEst; + }else{ + nRowEst = (iUpper-iLower)*p->aiRowEst[0]/SQLITE_INDEX_SAMPLES; + *pnRow = nRowEst; + } + +whereEqualScanEst_cancel: + sqlite3ValueFree(pRhs); + return rc; +} +#endif /* defined(SQLITE_ENABLE_STAT2) */ + +#ifdef SQLITE_ENABLE_STAT2 +/* +** Estimate the number of rows that will be returned based on +** an IN constraint where the right-hand side of the IN operator +** is a list of values. Example: +** +** WHERE x IN (1,2,3,4) +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereInScanEst( + Parse *pParse, /* Parsing & code generating context */ + Index *p, /* The index whose left-most column is pTerm */ + ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ + double *pnRow /* Write the revised row estimate here */ +){ + sqlite3_value *pVal = 0; /* One value from list */ + int iLower, iUpper; /* Range of histogram regions containing pRhs */ + u8 aff; /* Column affinity */ + int rc = SQLITE_OK; /* Subfunction return code */ + double nRowEst; /* New estimate of the number of rows */ + int nSpan = 0; /* Number of histogram regions spanned */ + int nSingle = 0; /* Histogram regions hit by a single value */ + int nNotFound = 0; /* Count of values that are not constants */ + int i; /* Loop counter */ + u8 aSpan[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions that are spanned */ + u8 aSingle[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions hit once */ + + assert( p->aSample!=0 ); + aff = p->pTable->aCol[p->aiColumn[0]].affinity; + memset(aSpan, 0, sizeof(aSpan)); + memset(aSingle, 0, sizeof(aSingle)); + for(i=0; inExpr; i++){ + sqlite3ValueFree(pVal); + rc = valueFromExpr(pParse, pList->a[i].pExpr, aff, &pVal); + if( rc ) break; + if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ + nNotFound++; + continue; + } + rc = whereRangeRegion(pParse, p, pVal, 0, &iLower); + if( rc ) break; + rc = whereRangeRegion(pParse, p, pVal, 1, &iUpper); + if( rc ) break; + if( iLower>=iUpper ){ + aSingle[iLower] = 1; + }else{ + assert( iLower>=0 && iUpper<=SQLITE_INDEX_SAMPLES ); + while( iLoweraiRowEst[0]/(2*SQLITE_INDEX_SAMPLES) + + nNotFound*p->aiRowEst[1]; + if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + *pnRow = nRowEst; + WHERETRACE(("IN row estimate: nSpan=%d, nSingle=%d, nNotFound=%d, est=%g\n", + nSpan, nSingle, nNotFound, nRowEst)); + } + sqlite3ValueFree(pVal); + return rc; +} +#endif /* defined(SQLITE_ENABLE_STAT2) */ + + +/* +** Find the best query plan for accessing a particular table. Write the +** best query plan and its cost into the WhereCost object supplied as the +** last parameter. +** +** The lowest cost plan wins. The cost is an estimate of the amount of +** CPU and disk I/O needed to process the requested result. +** Factors that influence cost include: +** +** * The estimated number of rows that will be retrieved. (The +** fewer the better.) +** +** * Whether or not sorting must occur. +** +** * Whether or not there must be separate lookups in the +** index and in the main table. +** +** If there was an INDEXED BY clause (pSrc->pIndex) attached to the table in +** the SQL statement, then this function only considers plans using the +** named index. If no such plan is found, then the returned cost is +** SQLITE_BIG_DBL. If a plan is found that uses the named index, +** then the cost is calculated in the usual way. +** +** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table +** in the SELECT statement, then no indexes are considered. However, the +** selected plan may still take advantage of the built-in rowid primary key +** index. +*/ +static void bestBtreeIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to search */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ + ExprList *pOrderBy, /* The ORDER BY clause */ + WhereCost *pCost /* Lowest cost query plan */ +){ + int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ + Index *pProbe; /* An index we are evaluating */ + Index *pIdx; /* Copy of pProbe, or zero for IPK index */ + int eqTermMask; /* Current mask of valid equality operators */ + int idxEqTermMask; /* Index mask of valid equality operators */ + Index sPk; /* A fake index object for the primary key */ + unsigned int aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ + + /* Initialize the cost to a worst-case value */ + memset(pCost, 0, sizeof(*pCost)); + pCost->rCost = SQLITE_BIG_DBL; + + /* If the pSrc table is the right table of a LEFT JOIN then we may not + ** use an index to satisfy IS NULL constraints on that table. This is + ** because columns might end up being NULL if the table does not match - + ** a circumstance which the index cannot help us discover. Ticket #2177. + */ + if( pSrc->jointype & JT_LEFT ){ + idxEqTermMask = WO_EQ|WO_IN; + }else{ + idxEqTermMask = WO_EQ|WO_IN|WO_ISNULL; + } + + if( pSrc->pIndex ){ + /* An INDEXED BY clause specifies a particular index to use */ + pIdx = pProbe = pSrc->pIndex; + wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); + eqTermMask = idxEqTermMask; + }else{ + /* There is no INDEXED BY clause. Create a fake Index object in local + ** variable sPk to represent the rowid primary key index. Make this + ** fake index the first in a chain of Index objects with all of the real + ** indices to follow */ + Index *pFirst; /* First of real indices on the table */ + memset(&sPk, 0, sizeof(Index)); + sPk.nColumn = 1; + sPk.aiColumn = &aiColumnPk; + sPk.aiRowEst = aiRowEstPk; + sPk.onError = OE_Replace; + sPk.pTable = pSrc->pTab; + aiRowEstPk[0] = pSrc->pTab->nRowEst; + aiRowEstPk[1] = 1; + pFirst = pSrc->pTab->pIndex; + if( pSrc->notIndexed==0 ){ + /* The real indices of the table are only considered if the + ** NOT INDEXED qualifier is omitted from the FROM clause */ + sPk.pNext = pFirst; + } + pProbe = &sPk; + wsFlagMask = ~( + WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE + ); + eqTermMask = WO_EQ|WO_IN; + pIdx = 0; + } + + /* Loop over all indices looking for the best one to use + */ + for(; pProbe; pIdx=pProbe=pProbe->pNext){ + const unsigned int * const aiRowEst = pProbe->aiRowEst; + double cost; /* Cost of using pProbe */ + double nRow; /* Estimated number of rows in result set */ + double log10N; /* base-10 logarithm of nRow (inexact) */ + int rev; /* True to scan in reverse order */ + int wsFlags = 0; + Bitmask used = 0; + + /* The following variables are populated based on the properties of + ** index being evaluated. They are then used to determine the expected + ** cost and number of rows returned. + ** + ** nEq: + ** Number of equality terms that can be implemented using the index. + ** In other words, the number of initial fields in the index that + ** are used in == or IN or NOT NULL constraints of the WHERE clause. + ** + ** nInMul: + ** The "in-multiplier". This is an estimate of how many seek operations + ** SQLite must perform on the index in question. For example, if the + ** WHERE clause is: + ** + ** WHERE a IN (1, 2, 3) AND b IN (4, 5, 6) + ** + ** SQLite must perform 9 lookups on an index on (a, b), so nInMul is + ** set to 9. Given the same schema and either of the following WHERE + ** clauses: + ** + ** WHERE a = 1 + ** WHERE a >= 2 + ** + ** nInMul is set to 1. + ** + ** If there exists a WHERE term of the form "x IN (SELECT ...)", then + ** the sub-select is assumed to return 25 rows for the purposes of + ** determining nInMul. + ** + ** bInEst: + ** Set to true if there was at least one "x IN (SELECT ...)" term used + ** in determining the value of nInMul. Note that the RHS of the + ** IN operator must be a SELECT, not a value list, for this variable + ** to be true. + ** + ** estBound: + ** An estimate on the amount of the table that must be searched. A + ** value of 100 means the entire table is searched. Range constraints + ** might reduce this to a value less than 100 to indicate that only + ** a fraction of the table needs searching. In the absence of + ** sqlite_stat2 ANALYZE data, a single inequality reduces the search + ** space to 1/4rd its original size. So an x>? constraint reduces + ** estBound to 25. Two constraints (x>? AND xnColumn; nEq++){ + int j = pProbe->aiColumn[nEq]; + pTerm = findTerm(pWC, iCur, j, notReady, eqTermMask, pIdx); + if( pTerm==0 ) break; + wsFlags |= (WHERE_COLUMN_EQ|WHERE_ROWID_EQ); + if( pTerm->eOperator & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ + nInMul *= 25; + bInEst = 1; + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nInMul *= pExpr->x.pList->nExpr; + } + }else if( pTerm->eOperator & WO_ISNULL ){ + wsFlags |= WHERE_COLUMN_NULL; + } +#ifdef SQLITE_ENABLE_STAT2 + if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; +#endif + used |= pTerm->prereqRight; + } + + /* Determine the value of estBound. */ + if( nEqnColumn && pProbe->bUnordered==0 ){ + int j = pProbe->aiColumn[nEq]; + if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ + WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); + WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); + whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &estBound); + if( pTop ){ + nBound = 1; + wsFlags |= WHERE_TOP_LIMIT; + used |= pTop->prereqRight; + } + if( pBtm ){ + nBound++; + wsFlags |= WHERE_BTM_LIMIT; + used |= pBtm->prereqRight; + } + wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); + } + }else if( pProbe->onError!=OE_None ){ + testcase( wsFlags & WHERE_COLUMN_IN ); + testcase( wsFlags & WHERE_COLUMN_NULL ); + if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ + wsFlags |= WHERE_UNIQUE; + } + } + + /* If there is an ORDER BY clause and the index being considered will + ** naturally scan rows in the required order, set the appropriate flags + ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index + ** will scan rows in a different order, set the bSort variable. */ + if( pOrderBy ){ + if( (wsFlags & WHERE_COLUMN_IN)==0 + && pProbe->bUnordered==0 + && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, + nEq, wsFlags, &rev) + ){ + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + wsFlags |= (rev ? WHERE_REVERSE : 0); + }else{ + bSort = 1; + } + } + + /* If currently calculating the cost of using an index (not the IPK + ** index), determine if all required column data may be obtained without + ** using the main table (i.e. if the index is a covering + ** index for this query). If it is, set the WHERE_IDX_ONLY flag in + ** wsFlags. Otherwise, set the bLookup variable to true. */ + if( pIdx && wsFlags ){ + Bitmask m = pSrc->colUsed; + int j; + for(j=0; jnColumn; j++){ + int x = pIdx->aiColumn[j]; + if( xaiRowEst[0] ){ + nRow = aiRowEst[0]/2; + nInMul = (int)(nRow / aiRowEst[nEq]); + } + +#ifdef SQLITE_ENABLE_STAT2 + /* If the constraint is of the form x=VALUE and histogram + ** data is available for column x, then it might be possible + ** to get a better estimate on the number of rows based on + ** VALUE and how common that value is according to the histogram. + */ + if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){ + if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ + testcase( pFirstTerm->eOperator==WO_EQ ); + testcase( pFirstTerm->eOperator==WO_ISNULL ); + whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); + }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){ + whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); + } + } +#endif /* SQLITE_ENABLE_STAT2 */ + + /* Adjust the number of output rows and downward to reflect rows + ** that are excluded by range constraints. + */ + nRow = (nRow * (double)estBound) / (double)100; + if( nRow<1 ) nRow = 1; + + /* Experiments run on real SQLite databases show that the time needed + ** to do a binary search to locate a row in a table or index is roughly + ** log10(N) times the time to move from one row to the next row within + ** a table or index. The actual times can vary, with the size of + ** records being an important factor. Both moves and searches are + ** slower with larger records, presumably because fewer records fit + ** on one page and hence more pages have to be fetched. + ** + ** The ANALYZE command and the sqlite_stat1 and sqlite_stat2 tables do + ** not give us data on the relative sizes of table and index records. + ** So this computation assumes table records are about twice as big + ** as index records + */ + if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + /* The cost of a full table scan is a number of move operations equal + ** to the number of rows in the table. + ** + ** We add an additional 4x penalty to full table scans. This causes + ** the cost function to err on the side of choosing an index over + ** choosing a full scan. This 4x full-scan penalty is an arguable + ** decision and one which we expect to revisit in the future. But + ** it seems to be working well enough at the moment. + */ + cost = aiRowEst[0]*4; + }else{ + log10N = estLog(aiRowEst[0]); + cost = nRow; + if( pIdx ){ + if( bLookup ){ + /* For an index lookup followed by a table lookup: + ** nInMul index searches to find the start of each index range + ** + nRow steps through the index + ** + nRow table searches to lookup the table entry using the rowid + */ + cost += (nInMul + nRow)*log10N; + }else{ + /* For a covering index: + ** nInMul index searches to find the initial entry + ** + nRow steps through the index + */ + cost += nInMul*log10N; + } + }else{ + /* For a rowid primary key lookup: + ** nInMult table searches to find the initial entry for each range + ** + nRow steps through the table + */ + cost += nInMul*log10N; + } + } + + /* Add in the estimated cost of sorting the result. Actual experimental + ** measurements of sorting performance in SQLite show that sorting time + ** adds C*N*log10(N) to the cost, where N is the number of rows to be + ** sorted and C is a factor between 1.95 and 4.3. We will split the + ** difference and select C of 3.0. + */ + if( bSort ){ + cost += nRow*estLog(nRow)*3; + } + + /**** Cost of using this index has now been computed ****/ + + /* If there are additional constraints on this table that cannot + ** be used with the current index, but which might lower the number + ** of output rows, adjust the nRow value accordingly. This only + ** matters if the current index is the least costly, so do not bother + ** with this step if we already know this index will not be chosen. + ** Also, never reduce the output row count below 2 using this step. + ** + ** It is critical that the notValid mask be used here instead of + ** the notReady mask. When computing an "optimal" index, the notReady + ** mask will only have one bit set - the bit for the current table. + ** The notValid mask, on the other hand, always has all bits set for + ** tables that are not in outer loops. If notReady is used here instead + ** of notValid, then a optimal index that depends on inner joins loops + ** might be selected even when there exists an optimal index that has + ** no such dependency. + */ + if( nRow>2 && cost<=pCost->rCost ){ + int k; /* Loop counter */ + int nSkipEq = nEq; /* Number of == constraints to skip */ + int nSkipRange = nBound; /* Number of < constraints to skip */ + Bitmask thisTab; /* Bitmap for pSrc */ + + thisTab = getMask(pWC->pMaskSet, iCur); + for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ + if( pTerm->wtFlags & TERM_VIRTUAL ) continue; + if( (pTerm->prereqAll & notValid)!=thisTab ) continue; + if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ + if( nSkipEq ){ + /* Ignore the first nEq equality matches since the index + ** has already accounted for these */ + nSkipEq--; + }else{ + /* Assume each additional equality match reduces the result + ** set size by a factor of 10 */ + nRow /= 10; + } + }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ + if( nSkipRange ){ + /* Ignore the first nSkipRange range constraints since the index + ** has already accounted for these */ + nSkipRange--; + }else{ + /* Assume each additional range constraint reduces the result + ** set size by a factor of 3. Indexed range constraints reduce + ** the search space by a larger factor: 4. We make indexed range + ** more selective intentionally because of the subjective + ** observation that indexed range constraints really are more + ** selective in practice, on average. */ + nRow /= 3; + } + }else if( pTerm->eOperator!=WO_NOOP ){ + /* Any other expression lowers the output row count by half */ + nRow /= 2; + } + } + if( nRow<2 ) nRow = 2; + } + + + WHERETRACE(( + "%s(%s): nEq=%d nInMul=%d estBound=%d bSort=%d bLookup=%d wsFlags=0x%x\n" + " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", + pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), + nEq, nInMul, estBound, bSort, bLookup, wsFlags, + notReady, log10N, nRow, cost, used + )); + + /* If this index is the best we have seen so far, then record this + ** index and its cost in the pCost structure. + */ + if( (!pIdx || wsFlags) + && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) + ){ + pCost->rCost = cost; + pCost->used = used; + pCost->plan.nRow = nRow; + pCost->plan.wsFlags = (wsFlags&wsFlagMask); + pCost->plan.nEq = nEq; + pCost->plan.u.pIdx = pIdx; + } + + /* If there was an INDEXED BY clause, then only that one index is + ** considered. */ + if( pSrc->pIndex ) break; + + /* Reset masks for the next index in the loop */ + wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE); + eqTermMask = idxEqTermMask; + } + + /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag + ** is set, then reverse the order that the index will be scanned + ** in. This is used for application testing, to help find cases + ** where application behaviour depends on the (undefined) order that + ** SQLite outputs rows in in the absence of an ORDER BY clause. */ + if( !pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ + pCost->plan.wsFlags |= WHERE_REVERSE; + } + + assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); + assert( pCost->plan.u.pIdx==0 || (pCost->plan.wsFlags&WHERE_ROWID_EQ)==0 ); + assert( pSrc->pIndex==0 + || pCost->plan.u.pIdx==0 + || pCost->plan.u.pIdx==pSrc->pIndex + ); + + WHERETRACE(("best index is: %s\n", + ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : + pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") + )); + + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); + pCost->plan.wsFlags |= eqTermMask; +} + +/* +** Find the query plan for accessing table pSrc->pTab. Write the +** best query plan and its cost into the WhereCost object supplied +** as the last parameter. This function may calculate the cost of +** both real and virtual table scans. +*/ +static void bestIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to search */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ + ExprList *pOrderBy, /* The ORDER BY clause */ + WhereCost *pCost /* Lowest cost query plan */ +){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pSrc->pTab) ){ + sqlite3_index_info *p = 0; + bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); + if( p->needToFreeIdxStr ){ + sqlite3_free(p->idxStr); + } + sqlite3DbFree(pParse->db, p); + }else +#endif + { + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + } +} + +/* +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. +** +** Consider the term t2.z='ok' in the following queries: +** +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' +** +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** IMPLEMENTATION-OF: R-24597-58655 No tests are done for terms that are +** completely satisfied by indices. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +*/ +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + if( pTerm + && (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + pTerm->wtFlags |= TERM_CODED; + if( pTerm->iParent>=0 ){ + WhereTerm *pOther = &pTerm->pWC->a[pTerm->iParent]; + if( (--pOther->nChild)==0 ){ + disableTerm(pLevel, pOther); + } + } + } +} + +/* +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. +** +** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_NONE, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. +*/ +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; + } + assert( v!=0 ); + + /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ + n--; + base++; + zAff++; + } + while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ + n--; + } + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp2(v, OP_Affinity, base, n); + sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); + } +} + + +/* +** Generate code for a single equality term of the WHERE clause. An equality +** term can be either X=expr or X IN (...). pTerm is the term to be +** coded. +** +** The current value for the constraint is left in register iReg. +** +** For a constraint of the form X=expr, the expression is evaluated and its +** result is left on the stack. For constraints of the form X IN (...) +** this routine sets up a loop that will iterate over all values of X. +*/ +static int codeEqualityTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* When level of the FROM clause we are working on */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ + + assert( iTarget>0 ); + if( pX->op==TK_EQ ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY + }else{ + int eType; + int iTab; + struct InLoop *pIn; + + assert( pX->op==TK_IN ); + iReg = iTarget; + eType = sqlite3FindInIndex(pParse, pX, 0); + iTab = pX->iTable; + sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + } + pLevel->u.in.nIn++; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + pIn += pLevel->u.in.nIn - 1; + pIn->iCur = iTab; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); + }else{ + pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); + } + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); + }else{ + pLevel->u.in.nIn = 0; + } +#endif + } + disableTerm(pLevel, pTerm); + return iReg; +} + +/* +** Generate code that will evaluate all == and IN constraints for an +** index. +** +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. +** +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. +** +** This routine always allocates at least one memory cell and returns +** the index of that memory cell. The code that +** calls this routine will use that memory cell to store the termination +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use NONE affinity are set to +** SQLITE_AFF_NONE. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_NONE. +*/ +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + WhereClause *pWC, /* The WHERE clause */ + Bitmask notReady, /* Which parts of FROM have not yet been coded */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ +){ + int nEq = pLevel->plan.nEq; /* The number of == or IN constraints to code */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + int iCur = pLevel->iTabCur; /* The cursor of the table */ + WhereTerm *pTerm; /* A single constraint term */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ + + /* This module is only called on query plans that use an index. */ + assert( pLevel->plan.wsFlags & WHERE_INDEXED ); + pIdx = pLevel->plan.u.pIdx; + + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLevel->plan.nEq + nExtraReg; + pParse->nMem += nReg; + + zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); + if( !zAff ){ + pParse->db->mallocFailed = 1; + } + + /* Evaluate the equality constraints + */ + assert( pIdx->nColumn>=nEq ); + for(j=0; jaiColumn[j]; + pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); + if( NEVER(pTerm==0) ) break; + /* The following true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + } + } + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IN ); + if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk); + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ + zAff[j] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_NONE; + } + } + } + } + *pzAff = zAff; + return regBase; +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. +*/ +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppend(pStr, zColumn, -1); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); +} + +/* +** Argument pLevel describes a strategy for scanning table pTab. This +** function returns a pointer to a string buffer containing a description +** of the subset of table rows scanned by the strategy in the form of an +** SQL expression. Or, if all rows are scanned, NULL is returned. +** +** For example, if the query: +** +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" +** +** The returned pointer points to memory obtained from sqlite3DbMalloc(). +** It is the responsibility of the caller to free the buffer when it is +** no longer required. +*/ +static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ + WherePlan *pPlan = &pLevel->plan; + Index *pIndex = pPlan->u.pIdx; + int nEq = pPlan->nEq; + int i, j; + Column *aCol = pTab->aCol; + int *aiColumn = pIndex->aiColumn; + StrAccum txt; + + if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ + return 0; + } + sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); + txt.db = db; + sqlite3StrAccumAppend(&txt, " (", 2); + for(i=0; i"); + } + if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ + explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<"); + } + sqlite3StrAccumAppend(&txt, ")", 1); + return sqlite3StrAccumFinish(&txt); +} + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single +** record is added to the output to describe the table scan strategy in +** pLevel. +*/ +static void explainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + if( pParse->explain==2 ){ + u32 flags = pLevel->plan.wsFlags; + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zMsg; /* Text to add to EQP output */ + sqlite3_int64 nRow; /* Expected number of rows visited by scan */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + + isSearch = (pLevel->plan.nEq>0) + || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + if( pItem->pSelect ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + }else{ + zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + } + + if( pItem->zAlias ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + } + if( (flags & WHERE_INDEXED)!=0 ){ + char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); + zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, + ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), + ((flags & WHERE_IDX_ONLY)?"COVERING ":""), + ((flags & WHERE_TEMP_INDEX)?"":" "), + ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName), + zWhere + ); + sqlite3DbFree(db, zWhere); + }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + + if( flags&WHERE_ROWID_EQ ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); + }else if( flags&WHERE_TOP_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidplan.u.pVtabIdx; + zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, + pVtabIdx->idxNum, pVtabIdx->idxStr); + } +#endif + if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ + testcase( wctrlFlags & WHERE_ORDERBY_MIN ); + nRow = 1; + }else{ + nRow = (sqlite3_int64)pLevel->plan.nRow; + } + zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); + sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + } +} +#else +# define explainOneScan(u,v,w,x,y,z) +#endif /* SQLITE_OMIT_EXPLAIN */ + + +/* +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. +*/ +static Bitmask codeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ + Bitmask notReady /* Which tables are currently available */ +){ + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ + int bRev; /* True if we need to scan in reverse order */ + WhereLevel *pLevel; /* The where level to be coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + Parse *pParse; /* Parsing context */ + Vdbe *v; /* The prepared stmt under constructions */ + struct SrcList_item *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ + + pParse = pWInfo->pParse; + v = pParse->pVdbe; + pWC = pWInfo->pWC; + pLevel = &pWInfo->a[iLevel]; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; + omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 + && (wctrlFlags & WHERE_FORCE_TABLE)==0; + + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 0: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; + int nConstraint = pVtabIdx->nConstraint; + struct sqlite3_index_constraint_usage *aUsage = + pVtabIdx->aConstraintUsage; + const struct sqlite3_index_constraint *aConstraint = + pVtabIdx->aConstraint; + + sqlite3ExprCachePush(pParse); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + for(j=1; j<=nConstraint; j++){ + for(k=0; ka[iTerm].pExpr->pRight, iReg+j+1); + break; + } + } + if( k==nConstraint ) break; + } + sqlite3VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, j-1, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, + pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); + pVtabIdx->needToFreeIdxStr = 0; + for(j=0; ja[iTerm]); + } + } + pLevel->op = OP_VNext; + pLevel->p1 = iCur; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + sqlite3ExprCachePop(pParse, 1); + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + if( pLevel->plan.wsFlags & WHERE_ROWID_EQ ){ + /* Case 1: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + iReleaseReg = sqlite3GetTempReg(pParse); + pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( pTerm->leftCursor==iCur ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){ + /* Case 2: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; + + assert( omitTable==0 ); + pStart = findTerm(pWC, iCur, -1, notReady, WO_GT|WO_GE, 0); + pEnd = findTerm(pWC, iCur, -1, notReady, WO_LT|WO_LE, 0); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ + + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGt, + /* TK_LE */ OP_SeekLe, + /* TK_LT */ OP_SeekLt, + /* TK_GE */ OP_SeekGe + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + + testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + pX = pStart->pExpr; + assert( pX!=0 ); + assert( pStart->leftCursor==iCur ); + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + VdbeComment((v, "pk")); + sqlite3ExprCacheAffinityChange(pParse, r1, 1); + sqlite3ReleaseTempReg(pParse, rTemp); + disableTerm(pLevel, pStart); + }else{ + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( pEnd->leftCursor==iCur ); + testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + memEndValue = ++pParse->nMem; + sqlite3ExprCode(pParse, pX->pRight, memEndValue); + if( pX->op==TK_LT || pX->op==TK_GT ){ + testOp = bRev ? OP_Le : OP_Ge; + }else{ + testOp = bRev ? OP_Lt : OP_Gt; + } + disableTerm(pLevel, pEnd); + } + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + if( pStart==0 && pEnd==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } + if( testOp!=OP_Noop ){ + iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){ + /* Case 3: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGt, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLt, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_Noop, /* 0: (!end_constraints) */ + OP_IdxGE, /* 1: (end_constraints && !bRev) */ + OP_IdxLT /* 2: (end_constraints && bRev) */ + }; + int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ + int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ + int regBase; /* Base register holding constraint values */ + int r1; /* Temp register */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char *zEndAff; /* Affinity for end of range constraint */ + + pIdx = pLevel->plan.u.pIdx; + iIdxCur = pLevel->iIdxCur; + k = pIdx->aiColumn[nEq]; /* Column for inequality constraints */ + + /* If this loop satisfies a sort order (pOrderBy) request that + ** was passed to this function to implement a "SELECT min(x) ..." + ** query, then the caller will only allow the loop to run for + ** a single iteration. This means that the first row returned + ** should not have a NULL value stored in 'x'. If column 'x' is + ** the first one after the nEq equality constraints in the index, + ** this requires some special handling. + */ + if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && (pLevel->plan.wsFlags&WHERE_ORDERBY) + && (pIdx->nColumn>nEq) + ){ + /* assert( pOrderBy->nExpr==1 ); */ + /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */ + isMinQuery = 1; + nExtraReg = 1; + } + + /* Find any inequality constraint terms for the start and end + ** of the range. + */ + if( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = findTerm(pWC, iCur, k, notReady, (WO_LT|WO_LE), pIdx); + nExtraReg = 1; + } + if( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = findTerm(pWC, iCur, k, notReady, (WO_GT|WO_GE), pIdx); + nExtraReg = 1; + } + + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + regBase = codeAllEqualityTerms( + pParse, pLevel, pWC, notReady, nExtraReg, &zStartAff + ); + zEndAff = sqlite3DbStrDup(pParse->db, zStartAff); + addrNxt = pLevel->addrNxt; + + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( nEqnColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC) ){ + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + } + + testcase( pRangeStart && pRangeStart->eOperator & WO_LE ); + testcase( pRangeStart && pRangeStart->eOperator & WO_GE ); + testcase( pRangeEnd && pRangeEnd->eOperator & WO_LE ); + testcase( pRangeEnd && pRangeEnd->eOperator & WO_GE ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; + + /* Seek the index cursor to the start of the range. */ + nConstraint = nEq; + if( pRangeStart ){ + Expr *pRight = pRangeStart->pExpr->pRight; + sqlite3ExprCode(pParse, pRight, regBase+nEq); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 ){ + sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + } + if( zStartAff ){ + if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_NONE. */ + zStartAff[nEq] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ + zStartAff[nEq] = SQLITE_AFF_NONE; + } + } + nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + }else if( isMinQuery ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + nConstraint++; + startEq = 0; + start_constraints = 1; + } + codeApplyAffinity(pParse, regBase, nConstraint, zStartAff); + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + testcase( op==OP_Rewind ); + testcase( op==OP_Last ); + testcase( op==OP_SeekGt ); + testcase( op==OP_SeekGe ); + testcase( op==OP_SeekLe ); + testcase( op==OP_SeekLt ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + + /* Load the value for the inequality constraint at the end of the + ** range (if any). + */ + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); + sqlite3ExprCode(pParse, pRight, regBase+nEq); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 ){ + sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + } + if( zEndAff ){ + if( sqlite3CompareAffinity(pRight, zEndAff[nEq])==SQLITE_AFF_NONE){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_NONE. */ + zEndAff[nEq] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zEndAff[nEq]) ){ + zEndAff[nEq] = SQLITE_AFF_NONE; + } + } + codeApplyAffinity(pParse, regBase, nEq+1, zEndAff); + nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + } + sqlite3DbFree(pParse->db, zStartAff); + sqlite3DbFree(pParse->db, zEndAff); + + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + + /* Check if the index cursor is past the end of the range. */ + op = aEndOp[(pRangeEnd || nEq) * (1 + bRev)]; + testcase( op==OP_Noop ); + testcase( op==OP_IdxGE ); + testcase( op==OP_IdxLT ); + if( op!=OP_Noop ){ + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0); + } + + /* If there are inequality constraints, check that the value + ** of the table column that the inequality contrains is not NULL. + ** If it is, jump to the next iteration of the loop. + */ + r1 = sqlite3GetTempReg(pParse); + testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); + testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); + if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); + sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); + } + sqlite3ReleaseTempReg(pParse, r1); + + /* Seek the table cursor, if required */ + disableTerm(pLevel, pRangeStart); + disableTerm(pLevel, pRangeEnd); + if( !omitTable ){ + iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ + } + + /* Record the instruction used to terminate the loop. Disable + ** WHERE clause terms made redundant by the index range scan. + */ + if( pLevel->plan.wsFlags & WHERE_UNIQUE ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; + } + pLevel->p1 = iIdxCur; + }else + +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ + /* Case 4: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; + + pTerm = pLevel->plan.u.pTerm; + assert( pTerm!=0 ); + assert( pTerm->eOperator==WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; + + /* Set up a new SrcList ni pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(pParse->db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (i16)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; + } + + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. + */ + if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + regRowset = ++pParse->nMem; + regRowid = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + /* Loop through table entries that match term pOrTerm. */ + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, + WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); + if( pSubWInfo ){ + explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + int r; + r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, + regRowid); + sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, + sqlite3VdbeCurrentAddr(v)+2, r, iSet); + } + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + } + } + } + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); + + if( pWInfo->nLevel>1 ) sqlite3StackFree(pParse->db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + + { + /* Case 5: There is no usable index. We must do a complete + ** scan of the entire table. + */ + static const u8 aStep[] = { OP_Next, OP_Prev }; + static const u8 aStart[] = { OP_Rewind, OP_Last }; + assert( bRev==0 || bRev==1 ); + assert( omitTable==0 ); + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } + notReady &= ~getMask(pWC->pMaskSet, iCur); + + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. + ** + ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through + ** the use of indices become tests that are evaluated against each row of + ** the relevant input tables. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; + } + + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. + */ + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + sqlite3ExprCacheClear(pParse); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; + } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; + } + } + sqlite3ReleaseTempReg(pParse, iReleaseReg); + + return notReady; +} + +#if defined(SQLITE_TEST) +/* +** The following variable holds a text description of query plan generated +** by the most recent call to sqlite3WhereBegin(). Each call to WhereBegin +** overwrites the previous. This information is used for testing and +** analysis only. +*/ +SQLITE_API char sqlite3_query_plan[BMS*2*40]; /* Text of the join */ +static int nQPlan = 0; /* Next free slow in _query_plan[] */ + +#endif /* SQLITE_TEST */ + + +/* +** Free a WhereInfo structure +*/ +static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ + if( ALWAYS(pWInfo) ){ + int i; + for(i=0; inLevel; i++){ + sqlite3_index_info *pInfo = pWInfo->a[i].pIdxInfo; + if( pInfo ){ + /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */ + if( pInfo->needToFreeIdxStr ){ + sqlite3_free(pInfo->idxStr); + } + sqlite3DbFree(db, pInfo); + } + if( pWInfo->a[i].plan.wsFlags & WHERE_TEMP_INDEX ){ + Index *pIdx = pWInfo->a[i].plan.u.pIdx; + if( pIdx ){ + sqlite3DbFree(db, pIdx->zColAff); + sqlite3DbFree(db, pIdx); + } + } + } + whereClauseClear(pWInfo->pWC); + sqlite3DbFree(db, pWInfo); + } +} + + +/* +** Generate the beginning of the loop used for WHERE clause processing. +** The return value is a pointer to an opaque structure that contains +** information needed to terminate the loop. Later, the calling routine +** should invoke sqlite3WhereEnd() with the return value of this function +** in order to complete the WHERE clause processing. +** +** If an error occurs, this routine returns NULL. +** +** The basic idea is to do a nested loop, one loop for each table in +** the FROM clause of a select. (INSERT and UPDATE statements are the +** same as a SELECT with only a single table in the FROM clause.) For +** example, if the SQL is this: +** +** SELECT * FROM t1, t2, t3 WHERE ...; +** +** Then the code generated is conceptually like the following: +** +** foreach row1 in t1 do \ Code generated +** foreach row2 in t2 do |-- by sqlite3WhereBegin() +** foreach row3 in t3 do / +** ... +** end \ Code generated +** end |-- by sqlite3WhereEnd() +** end / +** +** Note that the loops might not be nested in the order in which they +** appear in the FROM clause if a different order is better able to make +** use of indices. Note also that when the IN operator appears in +** the WHERE clause, it might result in additional nested loops for +** scanning through all values on the right-hand side of the IN. +** +** There are Btree cursors associated with each table. t1 uses cursor +** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. +** And so forth. This routine generates code to open those VDBE cursors +** and sqlite3WhereEnd() generates the code to close them. +** +** The code that sqlite3WhereBegin() generates leaves the cursors named +** in pTabList pointing at their appropriate entries. The [...] code +** can use OP_Column and OP_Rowid opcodes on these cursors to extract +** data from the various tables of the loop. +** +** If the WHERE clause is empty, the foreach loops must each scan their +** entire tables. Thus a three-way join is an O(N^3) operation. But if +** the tables have indices and there are terms in the WHERE clause that +** refer to those indices, a complete table scan can be avoided and the +** code will run much faster. Most of the work of this routine is checking +** to see if there are indices that can be used to speed up the loop. +** +** Terms of the WHERE clause are also used to limit which rows actually +** make it to the "..." in the middle of the loop. After each "foreach", +** terms of the WHERE clause that use only terms in that loop and outer +** loops are evaluated and if false a jump is made around all subsequent +** inner loops (or around the "..." if the test occurs within the inner- +** most loop) +** +** OUTER JOINS +** +** An outer join of tables t1 and t2 is conceptally coded as follows: +** +** foreach row1 in t1 do +** flag = 0 +** foreach row2 in t2 do +** start: +** ... +** flag = 1 +** end +** if flag==0 then +** move the row2 cursor to a null row +** goto start +** fi +** end +** +** ORDER BY CLAUSE PROCESSING +** +** *ppOrderBy is a pointer to the ORDER BY clause of a SELECT statement, +** if there is one. If there is no ORDER BY clause or if this routine +** is called from an UPDATE or DELETE statement, then ppOrderBy is NULL. +** +** If an index can be used so that the natural output order of the table +** scan is correct for the ORDER BY clause, then that index is used and +** *ppOrderBy is set to NULL. This is an optimization that prevents an +** unnecessary sort of the result set if an index appropriate for the +** ORDER BY clause already exists. +** +** If the where clause loops cannot be arranged to provide the correct +** output order, then the *ppOrderBy is unchanged. +*/ +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* A list of all tables to be scanned */ + Expr *pWhere, /* The WHERE clause */ + ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ +){ + int i; /* Loop counter */ + int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ + int nTabList; /* Number of elements in pTabList */ + WhereInfo *pWInfo; /* Will become the return value of this function */ + Vdbe *v = pParse->pVdbe; /* The virtual database engine */ + Bitmask notReady; /* Cursors that are not yet positioned */ + WhereMaskSet *pMaskSet; /* The expression mask set */ + WhereClause *pWC; /* Decomposition of the WHERE clause */ + struct SrcList_item *pTabItem; /* A single entry from pTabList */ + WhereLevel *pLevel; /* A single level in the pWInfo list */ + int iFrom; /* First unused FROM clause element */ + int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ + sqlite3 *db; /* Database connection */ + + /* The number of tables in the FROM clause is limited by the number of + ** bits in a Bitmask + */ + testcase( pTabList->nSrc==BMS ); + if( pTabList->nSrc>BMS ){ + sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); + return 0; + } + + /* This function normally generates a nested loop for all tables in + ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should + ** only generate code for the first table in pTabList and assume that + ** any cursors associated with subsequent tables are uninitialized. + */ + nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; + + /* Allocate and initialize the WhereInfo structure that will become the + ** return value. A single allocation is used to store the WhereInfo + ** struct, the contents of WhereInfo.a[], the WhereClause structure + ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte + ** field (type Bitmask) it must be aligned on an 8-byte boundary on + ** some architectures. Hence the ROUND8() below. + */ + db = pParse->db; + nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + pWInfo = sqlite3DbMallocZero(db, + nByteWInfo + + sizeof(WhereClause) + + sizeof(WhereMaskSet) + ); + if( db->mallocFailed ){ + sqlite3DbFree(db, pWInfo); + pWInfo = 0; + goto whereBeginError; + } + pWInfo->nLevel = nTabList; + pWInfo->pParse = pParse; + pWInfo->pTabList = pTabList; + pWInfo->iBreak = sqlite3VdbeMakeLabel(v); + pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; + pWInfo->wctrlFlags = wctrlFlags; + pWInfo->savedNQueryLoop = pParse->nQueryLoop; + pMaskSet = (WhereMaskSet*)&pWC[1]; + + /* Split the WHERE clause into separate subexpressions where each + ** subexpression is separated by an AND operator. + */ + initMaskSet(pMaskSet); + whereClauseInit(pWC, pParse, pMaskSet); + sqlite3ExprCodeConstants(pParse, pWhere); + whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + + /* Special case: a WHERE clause that is constant. Evaluate the + ** expression and either jump over all of the code or fall thru. + */ + if( pWhere && (nTabList==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ + sqlite3ExprIfFalse(pParse, pWhere, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pWhere = 0; + } + + /* Assign a bit from the bitmask to every term in the FROM clause. + ** + ** When assigning bitmask values to FROM clause cursors, it must be + ** the case that if X is the bitmask for the N-th FROM clause term then + ** the bitmask for all FROM clause terms to the left of the N-th term + ** is (X-1). An expression from the ON clause of a LEFT JOIN can use + ** its Expr.iRightJoinTable value to find the bitmask of the right table + ** of the join. Subtracting one from the right table bitmask gives a + ** bitmask for all tables to the left of the join. Knowing the bitmask + ** for all tables to the left of a left join is important. Ticket #3015. + ** + ** Configure the WhereClause.vmask variable so that bits that correspond + ** to virtual table cursors are set. This is used to selectively disable + ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful + ** with virtual tables. + ** + ** Note that bitmasks are created for all pTabList->nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_ONETABLE_ONLY flag is set. + */ + assert( pWC->vmask==0 && pMaskSet->n==0 ); + for(i=0; inSrc; i++){ + createMask(pMaskSet, pTabList->a[i].iCursor); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ + pWC->vmask |= ((Bitmask)1 << i); + } +#endif + } +#ifndef NDEBUG + { + Bitmask toTheLeft = 0; + for(i=0; inSrc; i++){ + Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); + assert( (m-1)==toTheLeft ); + toTheLeft |= m; + } + } +#endif + + /* Analyze all of the subexpressions. Note that exprAnalyze() might + ** add new virtual terms onto the end of the WHERE clause. We do not + ** want to analyze these virtual terms, so start analyzing at the end + ** and work forward so that the added virtual terms are never processed. + */ + exprAnalyzeAll(pTabList, pWC); + if( db->mallocFailed ){ + goto whereBeginError; + } + + /* Chose the best index to use for each table in the FROM clause. + ** + ** This loop fills in the following fields: + ** + ** pWInfo->a[].pIdx The index to use for this level of the loop. + ** pWInfo->a[].wsFlags WHERE_xxx flags associated with pIdx + ** pWInfo->a[].nEq The number of == and IN constraints + ** pWInfo->a[].iFrom Which term of the FROM clause is being coded + ** pWInfo->a[].iTabCur The VDBE cursor for the database table + ** pWInfo->a[].iIdxCur The VDBE cursor for the index + ** pWInfo->a[].pTerm When wsFlags==WO_OR, the OR-clause term + ** + ** This loop also figures out the nesting order of tables in the FROM + ** clause. + */ + notReady = ~(Bitmask)0; + andFlags = ~0; + WHERETRACE(("*** Optimizer Start ***\n")); + for(i=iFrom=0, pLevel=pWInfo->a; i=0 && bestJ<0; isOptimal--){ + Bitmask mask; /* Mask of tables not yet ready */ + for(j=iFrom, pTabItem=&pTabList->a[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; + if( j!=iFrom && doNotReorder ) break; + m = getMask(pMaskSet, pTabItem->iCursor); + if( (m & notReady)==0 ){ + if( j==iFrom ) iFrom++; + continue; + } + mask = (isOptimal ? m : notReady); + pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); + if( pTabItem->pIndex==0 ) nUnconstrained++; + + WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", + j, isOptimal)); + assert( pTabItem->pTab ); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTabItem->pTab) ){ + sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; + bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost, pp); + }else +#endif + { + bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost); + } + assert( isOptimal || (sCost.used¬Ready)==0 ); + + /* If an INDEXED BY clause is present, then the plan must use that + ** index if it uses any index at all */ + assert( pTabItem->pIndex==0 + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sCost.plan.u.pIdx==pTabItem->pIndex ); + + if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + notIndexed |= m; + } + + /* Conditions under which this table becomes the best so far: + ** + ** (1) The table must not depend on other tables that have not + ** yet run. + ** + ** (2) A full-table-scan plan cannot supercede indexed plan unless + ** the full-table-scan is an "optimal" plan as defined above. + ** + ** (3) All tables have an INDEXED BY clause or this table lacks an + ** INDEXED BY clause or this table uses the specific + ** index specified by its INDEXED BY clause. This rule ensures + ** that a best-so-far is always selected even if an impossible + ** combination of INDEXED BY clauses are given. The error + ** will be detected and relayed back to the application later. + ** The NEVER() comes about because rule (2) above prevents + ** An indexable full-table-scan from reaching rule (3). + ** + ** (4) The plan cost must be lower than prior plans or else the + ** cost must be the same and the number of rows must be lower. + */ + if( (sCost.used¬Ready)==0 /* (1) */ + && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ + || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) + && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ + || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || sCost.rCost=0 ); + assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); + WHERETRACE(("*** Optimizer selects table %d for loop %d" + " with cost=%g and nRow=%g\n", + bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); + if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ + *ppOrderBy = 0; + } + andFlags &= bestPlan.plan.wsFlags; + pLevel->plan = bestPlan.plan; + testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); + testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); + if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ + pLevel->iIdxCur = pParse->nTab++; + }else{ + pLevel->iIdxCur = -1; + } + notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); + pLevel->iFrom = (u8)bestJ; + if( bestPlan.plan.nRow>=(double)1 ){ + pParse->nQueryLoop *= bestPlan.plan.nRow; + } + + /* Check that if the table scanned by this loop iteration had an + ** INDEXED BY clause attached to it, that the named index is being + ** used for the scan. If not, then query compilation has failed. + ** Return an error. + */ + pIdx = pTabList->a[bestJ].pIndex; + if( pIdx ){ + if( (bestPlan.plan.wsFlags & WHERE_INDEXED)==0 ){ + sqlite3ErrorMsg(pParse, "cannot use index: %s", pIdx->zName); + goto whereBeginError; + }else{ + /* If an INDEXED BY clause is used, the bestIndex() function is + ** guaranteed to find the index specified in the INDEXED BY clause + ** if it find an index at all. */ + assert( bestPlan.plan.u.pIdx==pIdx ); + } + } + } + WHERETRACE(("*** Optimizer Finished ***\n")); + if( pParse->nErr || db->mallocFailed ){ + goto whereBeginError; + } + + /* If the total query only selects a single row, then the ORDER BY + ** clause is irrelevant. + */ + if( (andFlags & WHERE_UNIQUE)!=0 && ppOrderBy ){ + *ppOrderBy = 0; + } + + /* If the caller is an UPDATE or DELETE statement that is requesting + ** to use a one-pass algorithm, determine if this is appropriate. + ** The one-pass algorithm only works if the WHERE clause constraints + ** the statement to update a single row. + */ + assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_UNIQUE)!=0 ){ + pWInfo->okOnePass = 1; + pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; + } + + /* Open all tables in the pTabList and any indices selected for + ** searching those tables. + */ + sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ + notReady = ~(Bitmask)0; + pWInfo->nRowOut = (double)1; + for(i=0, pLevel=pWInfo->a; ia[pLevel->iFrom]; + pTab = pTabItem->pTab; + pLevel->iTabCur = pTabItem->iCursor; + pWInfo->nRowOut *= pLevel->plan.nRow; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ + /* Do nothing */ + }else +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + int iCur = pTabItem->iCursor; + sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else +#endif + if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OMIT_OPEN)==0 ){ + int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; + sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); + testcase( pTab->nCol==BMS-1 ); + testcase( pTab->nCol==BMS ); + if( !pWInfo->okOnePass && pTab->nColcolUsed; + int n = 0; + for(; b; b=b>>1, n++){} + sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, + SQLITE_INT_TO_PTR(n), P4_INT32); + assert( n<=pTab->nCol ); + } + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + } +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ + constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); + }else +#endif + if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ + Index *pIx = pLevel->plan.u.pIdx; + KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIx); + int iIdxCur = pLevel->iIdxCur; + assert( pIx->pSchema==pTab->pSchema ); + assert( iIdxCur>=0 ); + sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIx->tnum, iDb, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIx->zName)); + } + sqlite3CodeVerifySchema(pParse, iDb); + notReady &= ~getMask(pWC->pMaskSet, pTabItem->iCursor); + } + pWInfo->iTop = sqlite3VdbeCurrentAddr(v); + if( db->mallocFailed ) goto whereBeginError; + + /* Generate the code to do the search. Each iteration of the for + ** loop below generates code for a single nested loop of the VM + ** program. + */ + notReady = ~(Bitmask)0; + for(i=0; ia[i]; + explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); + notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); + pWInfo->iContinue = pLevel->addrCont; + } + +#ifdef SQLITE_TEST /* For testing and debugging use only */ + /* Record in the query plan information about the current table + ** and the index used to access it (if any). If the table itself + ** is not used, its name is just '{}'. If no index is used + ** the index is listed as "{}". If the primary key is used the + ** index name is '*'. + */ + for(i=0; ia[i]; + pTabItem = &pTabList->a[pLevel->iFrom]; + z = pTabItem->zAlias; + if( z==0 ) z = pTabItem->pTab->zName; + n = sqlite3Strlen30(z); + if( n+nQPlan < sizeof(sqlite3_query_plan)-10 ){ + if( pLevel->plan.wsFlags & WHERE_IDX_ONLY ){ + memcpy(&sqlite3_query_plan[nQPlan], "{}", 2); + nQPlan += 2; + }else{ + memcpy(&sqlite3_query_plan[nQPlan], z, n); + nQPlan += n; + } + sqlite3_query_plan[nQPlan++] = ' '; + } + testcase( pLevel->plan.wsFlags & WHERE_ROWID_EQ ); + testcase( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ); + if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + memcpy(&sqlite3_query_plan[nQPlan], "* ", 2); + nQPlan += 2; + }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ + n = sqlite3Strlen30(pLevel->plan.u.pIdx->zName); + if( n+nQPlan < sizeof(sqlite3_query_plan)-2 ){ + memcpy(&sqlite3_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); + nQPlan += n; + sqlite3_query_plan[nQPlan++] = ' '; + } + }else{ + memcpy(&sqlite3_query_plan[nQPlan], "{} ", 3); + nQPlan += 3; + } + } + while( nQPlan>0 && sqlite3_query_plan[nQPlan-1]==' ' ){ + sqlite3_query_plan[--nQPlan] = 0; + } + sqlite3_query_plan[nQPlan] = 0; + nQPlan = 0; +#endif /* SQLITE_TEST // Testing and debugging use only */ + + /* Record the continuation address in the WhereInfo structure. Then + ** clean up and return. + */ + return pWInfo; + + /* Jump here if malloc fails */ +whereBeginError: + if( pWInfo ){ + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + } + return 0; +} + +/* +** Generate the end of the WHERE loop. See comments on +** sqlite3WhereBegin() for additional information. +*/ +SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ + Parse *pParse = pWInfo->pParse; + Vdbe *v = pParse->pVdbe; + int i; + WhereLevel *pLevel; + SrcList *pTabList = pWInfo->pTabList; + sqlite3 *db = pParse->db; + + /* Generate loop termination code. + */ + sqlite3ExprCacheClear(pParse); + for(i=pWInfo->nLevel-1; i>=0; i--){ + pLevel = &pWInfo->a[i]; + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + if( pLevel->op!=OP_Noop ){ + sqlite3VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); + sqlite3VdbeChangeP5(v, pLevel->p5); + } + if( pLevel->plan.wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + struct InLoop *pIn; + int j; + sqlite3VdbeResolveLabel(v, pLevel->addrNxt); + for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ + sqlite3VdbeJumpHere(v, pIn->addrInTop+1); + sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite3VdbeJumpHere(v, pIn->addrInTop-1); + } + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->iLeftJoin ){ + int addr; + addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); + assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ); + if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + } + if( pLevel->iIdxCur>=0 ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } + if( pLevel->op==OP_Return ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + } + sqlite3VdbeJumpHere(v, addr); + } + } + + /* The "break" point is here, just past the end of the outer loop. + ** Set it. + */ + sqlite3VdbeResolveLabel(v, pWInfo->iBreak); + + /* Close all of the cursors that were opened by sqlite3WhereBegin. + */ + assert( pWInfo->nLevel==1 || pWInfo->nLevel==pTabList->nSrc ); + for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ + struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; + Table *pTab = pTabItem->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)==0 + && pTab->pSelect==0 + && (pWInfo->wctrlFlags & WHERE_OMIT_CLOSE)==0 + ){ + int ws = pLevel->plan.wsFlags; + if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); + } + if( (ws & WHERE_INDEXED)!=0 && (ws & WHERE_TEMP_INDEX)==0 ){ + sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); + } + } + + /* If this scan uses an index, make code substitutions to read data + ** from the index in preference to the table. Sometimes, this means + ** the table need never be read from. This is a performance boost, + ** as the vdbe level waits until the table is read before actually + ** seeking the table cursor to the record corresponding to the current + ** position in the index. + ** + ** Calls to the code generator in between sqlite3WhereBegin and + ** sqlite3WhereEnd will have created code that references the table + ** directly. This loop scans all that code looking for opcodes + ** that reference the table and converts them into opcodes that + ** reference the index. + */ + if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){ + int k, j, last; + VdbeOp *pOp; + Index *pIdx = pLevel->plan.u.pIdx; + + assert( pIdx!=0 ); + pOp = sqlite3VdbeGetOp(v, pWInfo->iTop); + last = sqlite3VdbeCurrentAddr(v); + for(k=pWInfo->iTop; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + for(j=0; jnColumn; j++){ + if( pOp->p2==pIdx->aiColumn[j] ){ + pOp->p2 = j; + pOp->p1 = pLevel->iIdxCur; + break; + } + } + assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + || jnColumn ); + }else if( pOp->opcode==OP_Rowid ){ + pOp->p1 = pLevel->iIdxCur; + pOp->opcode = OP_IdxRowid; + } + } + } + } + + /* Final cleanup + */ + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + return; +} + +/************** End of where.c ***********************************************/ +/************** Begin file parse.c *******************************************/ +/* Driver template for the LEMON parser generator. +** The author disclaims copyright to this source code. +** +** This version of "lempar.c" is modified, slightly, for use by SQLite. +** The only modifications are the addition of a couple of NEVER() +** macros to disable tests that are needed in the case of a general +** LALR(1) grammar but which are always false in the +** specific grammar used by SQLite. +*/ +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ + + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +#define yytestcase(X) testcase(X) + +/* +** An instance of this structure holds information about the +** LIMIT clause of a SELECT statement. +*/ +struct LimitVal { + Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ + Expr *pOffset; /* The OFFSET expression. NULL if there is none */ +}; + +/* +** An instance of this structure is used to store the LIKE, +** GLOB, NOT LIKE, and NOT GLOB operators. +*/ +struct LikeOp { + Token eOperator; /* "like" or "glob" or "regexp" */ + int not; /* True if the NOT keyword is present */ +}; + +/* +** An instance of the following structure describes the event of a +** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, +** TK_DELETE, or TK_INSTEAD. If the event is of the form +** +** UPDATE ON (a,b,c) +** +** Then the "b" IdList records the list "a,b,c". +*/ +struct TrigEvent { int a; IdList * b; }; + +/* +** An instance of this structure holds the ATTACH key and the key type. +*/ +struct AttachKey { int type; Token key; }; + + + /* This is a utility routine used to set the ExprSpan.zStart and + ** ExprSpan.zEnd values of pOut so that the span covers the complete + ** range of text beginning with pStart and going to the end of pEnd. + */ + static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ + pOut->zStart = pStart->z; + pOut->zEnd = &pEnd->z[pEnd->n]; + } + + /* Construct a new Expr object from a single identifier. Use the + ** new Expr to populate pOut. Set the span of pOut to be the identifier + ** that created the expression. + */ + static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ + pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); + pOut->zStart = pValue->z; + pOut->zEnd = &pValue->z[pValue->n]; + } + + /* This routine constructs a binary expression node out of two ExprSpan + ** objects and uses the result to populate a new ExprSpan object. + */ + static void spanBinaryExpr( + ExprSpan *pOut, /* Write the result here */ + Parse *pParse, /* The parsing context. Errors accumulate here */ + int op, /* The binary operation */ + ExprSpan *pLeft, /* The left operand */ + ExprSpan *pRight /* The right operand */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pOut->zStart = pLeft->zStart; + pOut->zEnd = pRight->zEnd; + } + + /* Construct an expression node for a unary postfix operator + */ + static void spanUnaryPostfix( + ExprSpan *pOut, /* Write the new expression node here */ + Parse *pParse, /* Parsing context to record errors */ + int op, /* The operator */ + ExprSpan *pOperand, /* The operand */ + Token *pPostOp /* The operand token for setting the span */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->zStart = pOperand->zStart; + pOut->zEnd = &pPostOp->z[pPostOp->n]; + } + + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( db->mallocFailed==0 && pY->op==TK_NULL ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } + } + + /* Construct an expression node for a unary prefix operator + */ + static void spanUnaryPrefix( + ExprSpan *pOut, /* Write the new expression node here */ + Parse *pParse, /* Parsing context to record errors */ + int op, /* The operator */ + ExprSpan *pOperand, /* The operand */ + Token *pPreOp /* The operand token for setting the span */ + ){ + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->zStart = pPreOp->z; + pOut->zEnd = pOperand->zEnd; + } +/* Next is all token values, in a form suitable for use by makeheaders. +** This section will be null unless lemon is run with the -m switch. +*/ +/* +** These constants (all generated automatically by the parser generator) +** specify the various kinds of tokens (terminals) that the parser +** understands. +** +** Each symbol here is a terminal symbol in the grammar. +*/ +/* Make sure the INTERFACE macro is defined. +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/* The next thing included is series of defines which control +** various aspects of the generated parser. +** YYCODETYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 terminals +** and nonterminals. "int" is used otherwise. +** YYNOCODE is a number of type YYCODETYPE which corresponds +** to no legal terminal or nonterminal number. This +** number is used to fill in empty slots of the hash +** table. +** YYFALLBACK If defined, this indicates that one or more tokens +** have fall-back values which should be used if the +** original value of the token will not parse. +** YYACTIONTYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 rules and +** states combined. "int" is used otherwise. +** sqlite3ParserTOKENTYPE is the data type used for minor tokens given +** directly to the parser from the tokenizer. +** YYMINORTYPE is the data type used for all minor tokens. +** This is typically a union of many types, one of +** which is sqlite3ParserTOKENTYPE. The entry in the union +** for base tokens is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser +** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +*/ +#define YYCODETYPE unsigned char +#define YYNOCODE 253 +#define YYACTIONTYPE unsigned short int +#define YYWILDCARD 67 +#define sqlite3ParserTOKENTYPE Token +typedef union { + int yyinit; + sqlite3ParserTOKENTYPE yy0; + int yy4; + struct TrigEvent yy90; + ExprSpan yy118; + TriggerStep* yy203; + u8 yy210; + struct {int value; int mask;} yy215; + SrcList* yy259; + struct LimitVal yy292; + Expr* yy314; + ExprList* yy322; + struct LikeOp yy342; + IdList* yy384; + Select* yy387; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define sqlite3ParserARG_SDECL Parse *pParse; +#define sqlite3ParserARG_PDECL ,Parse *pParse +#define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse +#define sqlite3ParserARG_STORE yypParser->pParse = pParse +#define YYNSTATE 630 +#define YYNRULE 329 +#define YYFALLBACK 1 +#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) +#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) +#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) + +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** +** N == YYNSTATE+YYNRULE A syntax error has occurred. +** +** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** +** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** slots in the yy_action[] table. +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as +** +** yy_action[ yy_shift_ofst[S] + X ] +** +** If the index value yy_shift_ofst[S]+X is out of range or if the value +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] +** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that yy_default[S] should be used instead. +** +** The formula above is for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of +** YY_SHIFT_USE_DFLT. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +*/ +#define YY_ACTTAB_COUNT (1557) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 313, 960, 186, 419, 2, 172, 627, 597, 55, 55, + /* 10 */ 55, 55, 48, 53, 53, 53, 53, 52, 52, 51, /* 20 */ 51, 51, 50, 238, 302, 283, 623, 622, 516, 515, /* 30 */ 590, 584, 55, 55, 55, 55, 282, 53, 53, 53, /* 40 */ 53, 52, 52, 51, 51, 51, 50, 238, 6, 56, @@ -102094,1390 +105363,2476 @@ static void yy_reduce( fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, yyRuleName[yyruleno]); } -#endif /* NDEBUG */ - - /* Silence complaints from purify about yygotominor being uninitialized - ** in some cases when it is copied into the stack after the following - ** switch. yygotominor is uninitialized when a rule reduces that does - ** not set the value of its left-hand side nonterminal. Leaving the - ** value of the nonterminal uninitialized is utterly harmless as long - ** as the value is never used. So really the only thing this code - ** accomplishes is to quieten purify. - ** - ** 2007-01-16: The wireshark project (www.wireshark.org) reports that - ** without this code, their parser segfaults. I'm not sure what there - ** parser is doing to make this happen. This is the second bug report - ** from wireshark this week. Clearly they are stressing Lemon in ways - ** that it has not been previously stressed... (SQLite ticket #2172) - */ - /*memset(&yygotominor, 0, sizeof(yygotominor));*/ - yygotominor = yyzerominor; - - - switch( yyruleno ){ - /* Beginning here are the reduction cases. A typical example - ** follows: - ** case 0: - ** #line - ** { ... } // User supplied code - ** #line - ** break; - */ - case 5: /* explain ::= */ -{ sqlite3BeginParse(pParse, 0); } +#endif /* NDEBUG */ + + /* Silence complaints from purify about yygotominor being uninitialized + ** in some cases when it is copied into the stack after the following + ** switch. yygotominor is uninitialized when a rule reduces that does + ** not set the value of its left-hand side nonterminal. Leaving the + ** value of the nonterminal uninitialized is utterly harmless as long + ** as the value is never used. So really the only thing this code + ** accomplishes is to quieten purify. + ** + ** 2007-01-16: The wireshark project (www.wireshark.org) reports that + ** without this code, their parser segfaults. I'm not sure what there + ** parser is doing to make this happen. This is the second bug report + ** from wireshark this week. Clearly they are stressing Lemon in ways + ** that it has not been previously stressed... (SQLite ticket #2172) + */ + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; + + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ + case 5: /* explain ::= */ +{ sqlite3BeginParse(pParse, 0); } + break; + case 6: /* explain ::= EXPLAIN */ +{ sqlite3BeginParse(pParse, 1); } + break; + case 7: /* explain ::= EXPLAIN QUERY PLAN */ +{ sqlite3BeginParse(pParse, 2); } + break; + case 8: /* cmdx ::= cmd */ +{ sqlite3FinishCoding(pParse); } + break; + case 9: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy4);} + break; + case 13: /* transtype ::= */ +{yygotominor.yy4 = TK_DEFERRED;} + break; + case 14: /* transtype ::= DEFERRED */ + case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); + case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); + case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); + case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); +{yygotominor.yy4 = yymsp[0].major;} + break; + case 17: /* cmd ::= COMMIT trans_opt */ + case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); +{sqlite3CommitTransaction(pParse);} + break; + case 19: /* cmd ::= ROLLBACK trans_opt */ +{sqlite3RollbackTransaction(pParse);} + break; + case 22: /* cmd ::= SAVEPOINT nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); +} + break; + case 23: /* cmd ::= RELEASE savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); +} + break; + case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); +} + break; + case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ +{ + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy4,0,0,yymsp[-2].minor.yy4); +} + break; + case 27: /* createkw ::= CREATE */ +{ + pParse->db->lookaside.bEnabled = 0; + yygotominor.yy0 = yymsp[0].minor.yy0; +} + break; + case 28: /* ifnotexists ::= */ + case 31: /* temp ::= */ yytestcase(yyruleno==31); + case 70: /* autoinc ::= */ yytestcase(yyruleno==70); + case 83: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==83); + case 85: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==85); + case 87: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==87); + case 98: /* defer_subclause_opt ::= */ yytestcase(yyruleno==98); + case 109: /* ifexists ::= */ yytestcase(yyruleno==109); + case 120: /* distinct ::= ALL */ yytestcase(yyruleno==120); + case 121: /* distinct ::= */ yytestcase(yyruleno==121); + case 222: /* between_op ::= BETWEEN */ yytestcase(yyruleno==222); + case 225: /* in_op ::= IN */ yytestcase(yyruleno==225); +{yygotominor.yy4 = 0;} + break; + case 29: /* ifnotexists ::= IF NOT EXISTS */ + case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); + case 71: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==71); + case 86: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==86); + case 108: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==108); + case 119: /* distinct ::= DISTINCT */ yytestcase(yyruleno==119); + case 223: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==223); + case 226: /* in_op ::= NOT IN */ yytestcase(yyruleno==226); +{yygotominor.yy4 = 1;} + break; + case 32: /* create_table_args ::= LP columnlist conslist_opt RP */ +{ + sqlite3EndTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0); +} + break; + case 33: /* create_table_args ::= AS select */ +{ + sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy387); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy387); +} + break; + case 36: /* column ::= columnid type carglist */ +{ + yygotominor.yy0.z = yymsp[-2].minor.yy0.z; + yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; +} + break; + case 37: /* columnid ::= nm */ +{ + sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); + yygotominor.yy0 = yymsp[0].minor.yy0; +} + break; + case 38: /* id ::= ID */ + case 39: /* id ::= INDEXED */ yytestcase(yyruleno==39); + case 40: /* ids ::= ID|STRING */ yytestcase(yyruleno==40); + case 41: /* nm ::= id */ yytestcase(yyruleno==41); + case 42: /* nm ::= STRING */ yytestcase(yyruleno==42); + case 43: /* nm ::= JOIN_KW */ yytestcase(yyruleno==43); + case 46: /* typetoken ::= typename */ yytestcase(yyruleno==46); + case 49: /* typename ::= ids */ yytestcase(yyruleno==49); + case 127: /* as ::= AS nm */ yytestcase(yyruleno==127); + case 128: /* as ::= ids */ yytestcase(yyruleno==128); + case 138: /* dbnm ::= DOT nm */ yytestcase(yyruleno==138); + case 147: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==147); + case 251: /* collate ::= COLLATE ids */ yytestcase(yyruleno==251); + case 260: /* nmnum ::= plus_num */ yytestcase(yyruleno==260); + case 261: /* nmnum ::= nm */ yytestcase(yyruleno==261); + case 262: /* nmnum ::= ON */ yytestcase(yyruleno==262); + case 263: /* nmnum ::= DELETE */ yytestcase(yyruleno==263); + case 264: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==264); + case 265: /* plus_num ::= plus_opt number */ yytestcase(yyruleno==265); + case 266: /* minus_num ::= MINUS number */ yytestcase(yyruleno==266); + case 267: /* number ::= INTEGER|FLOAT */ yytestcase(yyruleno==267); + case 285: /* trnm ::= nm */ yytestcase(yyruleno==285); +{yygotominor.yy0 = yymsp[0].minor.yy0;} + break; + case 45: /* type ::= typetoken */ +{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} + break; + case 47: /* typetoken ::= typename LP signed RP */ +{ + yygotominor.yy0.z = yymsp[-3].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} + break; + case 48: /* typetoken ::= typename LP signed COMMA signed RP */ +{ + yygotominor.yy0.z = yymsp[-5].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} + break; + case 50: /* typename ::= typename ids */ +{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + break; + case 57: /* ccons ::= DEFAULT term */ + case 59: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==59); +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy118);} + break; + case 58: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy118);} + break; + case 60: /* ccons ::= DEFAULT MINUS term */ +{ + ExprSpan v; + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy118.pExpr, 0, 0); + v.zStart = yymsp[-1].minor.yy0.z; + v.zEnd = yymsp[0].minor.yy118.zEnd; + sqlite3AddDefaultValue(pParse,&v); +} + break; + case 61: /* ccons ::= DEFAULT id */ +{ + ExprSpan v; + spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); + sqlite3AddDefaultValue(pParse,&v); +} + break; + case 63: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy4);} + break; + case 64: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy4,yymsp[0].minor.yy4,yymsp[-2].minor.yy4);} + break; + case 65: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy4,0,0,0,0);} + break; + case 66: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy118.pExpr);} + break; + case 67: /* ccons ::= REFERENCES nm idxlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy322,yymsp[0].minor.yy4);} + break; + case 68: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy4);} + break; + case 69: /* ccons ::= COLLATE ids */ +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + break; + case 72: /* refargs ::= */ +{ yygotominor.yy4 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 73: /* refargs ::= refargs refarg */ +{ yygotominor.yy4 = (yymsp[-1].minor.yy4 & ~yymsp[0].minor.yy215.mask) | yymsp[0].minor.yy215.value; } + break; + case 74: /* refarg ::= MATCH nm */ + case 75: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==75); +{ yygotominor.yy215.value = 0; yygotominor.yy215.mask = 0x000000; } + break; + case 76: /* refarg ::= ON DELETE refact */ +{ yygotominor.yy215.value = yymsp[0].minor.yy4; yygotominor.yy215.mask = 0x0000ff; } + break; + case 77: /* refarg ::= ON UPDATE refact */ +{ yygotominor.yy215.value = yymsp[0].minor.yy4<<8; yygotominor.yy215.mask = 0x00ff00; } + break; + case 78: /* refact ::= SET NULL */ +{ yygotominor.yy4 = OE_SetNull; /* EV: R-33326-45252 */} + break; + case 79: /* refact ::= SET DEFAULT */ +{ yygotominor.yy4 = OE_SetDflt; /* EV: R-33326-45252 */} + break; + case 80: /* refact ::= CASCADE */ +{ yygotominor.yy4 = OE_Cascade; /* EV: R-33326-45252 */} + break; + case 81: /* refact ::= RESTRICT */ +{ yygotominor.yy4 = OE_Restrict; /* EV: R-33326-45252 */} + break; + case 82: /* refact ::= NO ACTION */ +{ yygotominor.yy4 = OE_None; /* EV: R-33326-45252 */} + break; + case 84: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 99: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==99); + case 101: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==101); + case 104: /* resolvetype ::= raisetype */ yytestcase(yyruleno==104); +{yygotominor.yy4 = yymsp[0].minor.yy4;} + break; + case 88: /* conslist_opt ::= */ +{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} + break; + case 89: /* conslist_opt ::= COMMA conslist */ +{yygotominor.yy0 = yymsp[-1].minor.yy0;} + break; + case 94: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy322,yymsp[0].minor.yy4,yymsp[-2].minor.yy4,0);} + break; + case 95: /* tcons ::= UNIQUE LP idxlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy322,yymsp[0].minor.yy4,0,0,0,0);} + break; + case 96: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy118.pExpr);} + break; + case 97: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ +{ + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy322, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[-1].minor.yy4); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy4); +} + break; + case 100: /* onconf ::= */ +{yygotominor.yy4 = OE_Default;} + break; + case 102: /* orconf ::= */ +{yygotominor.yy210 = OE_Default;} + break; + case 103: /* orconf ::= OR resolvetype */ +{yygotominor.yy210 = (u8)yymsp[0].minor.yy4;} + break; + case 105: /* resolvetype ::= IGNORE */ +{yygotominor.yy4 = OE_Ignore;} + break; + case 106: /* resolvetype ::= REPLACE */ +{yygotominor.yy4 = OE_Replace;} + break; + case 107: /* cmd ::= DROP TABLE ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy259, 0, yymsp[-1].minor.yy4); +} + break; + case 110: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ +{ + sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy387, yymsp[-6].minor.yy4, yymsp[-4].minor.yy4); +} + break; + case 111: /* cmd ::= DROP VIEW ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy259, 1, yymsp[-1].minor.yy4); +} + break; + case 112: /* cmd ::= select */ +{ + SelectDest dest = {SRT_Output, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy387, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy387); +} + break; + case 113: /* select ::= oneselect */ +{yygotominor.yy387 = yymsp[0].minor.yy387;} + break; + case 114: /* select ::= select multiselect_op oneselect */ +{ + if( yymsp[0].minor.yy387 ){ + yymsp[0].minor.yy387->op = (u8)yymsp[-1].minor.yy4; + yymsp[0].minor.yy387->pPrior = yymsp[-2].minor.yy387; + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy387); + } + yygotominor.yy387 = yymsp[0].minor.yy387; +} + break; + case 116: /* multiselect_op ::= UNION ALL */ +{yygotominor.yy4 = TK_ALL;} + break; + case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +{ + yygotominor.yy387 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy322,yymsp[-5].minor.yy259,yymsp[-4].minor.yy314,yymsp[-3].minor.yy322,yymsp[-2].minor.yy314,yymsp[-1].minor.yy322,yymsp[-7].minor.yy4,yymsp[0].minor.yy292.pLimit,yymsp[0].minor.yy292.pOffset); +} + break; + case 122: /* sclp ::= selcollist COMMA */ + case 247: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==247); +{yygotominor.yy322 = yymsp[-1].minor.yy322;} + break; + case 123: /* sclp ::= */ + case 151: /* orderby_opt ::= */ yytestcase(yyruleno==151); + case 159: /* groupby_opt ::= */ yytestcase(yyruleno==159); + case 240: /* exprlist ::= */ yytestcase(yyruleno==240); + case 246: /* idxlist_opt ::= */ yytestcase(yyruleno==246); +{yygotominor.yy322 = 0;} + break; + case 124: /* selcollist ::= sclp expr as */ +{ + yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, yymsp[-1].minor.yy118.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yygotominor.yy322,&yymsp[-1].minor.yy118); +} + break; + case 125: /* selcollist ::= sclp STAR */ +{ + Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); + yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy322, p); +} + break; + case 126: /* selcollist ::= sclp nm DOT STAR */ +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); + Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, pDot); +} + break; + case 129: /* as ::= */ +{yygotominor.yy0.n = 0;} + break; + case 130: /* from ::= */ +{yygotominor.yy259 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy259));} + break; + case 131: /* from ::= FROM seltablist */ +{ + yygotominor.yy259 = yymsp[0].minor.yy259; + sqlite3SrcListShiftJoinType(yygotominor.yy259); +} + break; + case 132: /* stl_prefix ::= seltablist joinop */ +{ + yygotominor.yy259 = yymsp[-1].minor.yy259; + if( ALWAYS(yygotominor.yy259 && yygotominor.yy259->nSrc>0) ) yygotominor.yy259->a[yygotominor.yy259->nSrc-1].jointype = (u8)yymsp[0].minor.yy4; +} + break; + case 133: /* stl_prefix ::= */ +{yygotominor.yy259 = 0;} + break; + case 134: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +{ + yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + sqlite3SrcListIndexedBy(pParse, yygotominor.yy259, &yymsp[-2].minor.yy0); +} + break; + case 135: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +{ + yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy387,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + } break; - case 6: /* explain ::= EXPLAIN */ -{ sqlite3BeginParse(pParse, 1); } + case 136: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ +{ + if( yymsp[-6].minor.yy259==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy314==0 && yymsp[0].minor.yy384==0 ){ + yygotominor.yy259 = yymsp[-4].minor.yy259; + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy259); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy259,0,0,0,0,0,0,0); + yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + } + } break; - case 7: /* explain ::= EXPLAIN QUERY PLAN */ -{ sqlite3BeginParse(pParse, 2); } + case 137: /* dbnm ::= */ + case 146: /* indexed_opt ::= */ yytestcase(yyruleno==146); +{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} break; - case 8: /* cmdx ::= cmd */ -{ sqlite3FinishCoding(pParse); } + case 139: /* fullname ::= nm dbnm */ +{yygotominor.yy259 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; - case 9: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy4);} + case 140: /* joinop ::= COMMA|JOIN */ +{ yygotominor.yy4 = JT_INNER; } break; - case 13: /* transtype ::= */ -{yygotominor.yy4 = TK_DEFERRED;} + case 141: /* joinop ::= JOIN_KW JOIN */ +{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } break; - case 14: /* transtype ::= DEFERRED */ - case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); - case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); - case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); - case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); -{yygotominor.yy4 = yymsp[0].major;} + case 142: /* joinop ::= JOIN_KW nm JOIN */ +{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } break; - case 17: /* cmd ::= COMMIT trans_opt */ - case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); -{sqlite3CommitTransaction(pParse);} + case 143: /* joinop ::= JOIN_KW nm nm JOIN */ +{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } break; - case 19: /* cmd ::= ROLLBACK trans_opt */ -{sqlite3RollbackTransaction(pParse);} + case 144: /* on_opt ::= ON expr */ + case 155: /* sortitem ::= expr */ yytestcase(yyruleno==155); + case 162: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==162); + case 169: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==169); + case 235: /* case_else ::= ELSE expr */ yytestcase(yyruleno==235); + case 237: /* case_operand ::= expr */ yytestcase(yyruleno==237); +{yygotominor.yy314 = yymsp[0].minor.yy118.pExpr;} break; - case 22: /* cmd ::= SAVEPOINT nm */ + case 145: /* on_opt ::= */ + case 161: /* having_opt ::= */ yytestcase(yyruleno==161); + case 168: /* where_opt ::= */ yytestcase(yyruleno==168); + case 236: /* case_else ::= */ yytestcase(yyruleno==236); + case 238: /* case_operand ::= */ yytestcase(yyruleno==238); +{yygotominor.yy314 = 0;} + break; + case 148: /* indexed_opt ::= NOT INDEXED */ +{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} + break; + case 149: /* using_opt ::= USING LP inscollist RP */ + case 181: /* inscollist_opt ::= LP inscollist RP */ yytestcase(yyruleno==181); +{yygotominor.yy384 = yymsp[-1].minor.yy384;} + break; + case 150: /* using_opt ::= */ + case 180: /* inscollist_opt ::= */ yytestcase(yyruleno==180); +{yygotominor.yy384 = 0;} + break; + case 152: /* orderby_opt ::= ORDER BY sortlist */ + case 160: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==160); + case 239: /* exprlist ::= nexprlist */ yytestcase(yyruleno==239); +{yygotominor.yy322 = yymsp[0].minor.yy322;} + break; + case 153: /* sortlist ::= sortlist COMMA sortitem sortorder */ { - sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); + yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322,yymsp[-1].minor.yy314); + if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; } break; - case 23: /* cmd ::= RELEASE savepoint_opt nm */ + case 154: /* sortlist ::= sortitem sortorder */ { - sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); + yygotominor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy314); + if( yygotominor.yy322 && ALWAYS(yygotominor.yy322->a) ) yygotominor.yy322->a[0].sortOrder = (u8)yymsp[0].minor.yy4; } break; - case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + case 156: /* sortorder ::= ASC */ + case 158: /* sortorder ::= */ yytestcase(yyruleno==158); +{yygotominor.yy4 = SQLITE_SO_ASC;} + break; + case 157: /* sortorder ::= DESC */ +{yygotominor.yy4 = SQLITE_SO_DESC;} + break; + case 163: /* limit_opt ::= */ +{yygotominor.yy292.pLimit = 0; yygotominor.yy292.pOffset = 0;} + break; + case 164: /* limit_opt ::= LIMIT expr */ +{yygotominor.yy292.pLimit = yymsp[0].minor.yy118.pExpr; yygotominor.yy292.pOffset = 0;} + break; + case 165: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yygotominor.yy292.pLimit = yymsp[-2].minor.yy118.pExpr; yygotominor.yy292.pOffset = yymsp[0].minor.yy118.pExpr;} + break; + case 166: /* limit_opt ::= LIMIT expr COMMA expr */ +{yygotominor.yy292.pOffset = yymsp[-2].minor.yy118.pExpr; yygotominor.yy292.pLimit = yymsp[0].minor.yy118.pExpr;} + break; + case 167: /* cmd ::= DELETE FROM fullname indexed_opt where_opt */ { - sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy259, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy259,yymsp[0].minor.yy314); } break; - case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + case 170: /* cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy4,0,0,yymsp[-2].minor.yy4); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy259, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy322,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy259,yymsp[-1].minor.yy322,yymsp[0].minor.yy314,yymsp[-5].minor.yy210); } break; - case 27: /* createkw ::= CREATE */ + case 171: /* setlist ::= setlist COMMA nm EQ expr */ { - pParse->db->lookaside.bEnabled = 0; - yygotominor.yy0 = yymsp[0].minor.yy0; + yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[0].minor.yy118.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); } break; - case 28: /* ifnotexists ::= */ - case 31: /* temp ::= */ yytestcase(yyruleno==31); - case 70: /* autoinc ::= */ yytestcase(yyruleno==70); - case 83: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ yytestcase(yyruleno==83); - case 85: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==85); - case 87: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==87); - case 98: /* defer_subclause_opt ::= */ yytestcase(yyruleno==98); - case 109: /* ifexists ::= */ yytestcase(yyruleno==109); - case 120: /* distinct ::= ALL */ yytestcase(yyruleno==120); - case 121: /* distinct ::= */ yytestcase(yyruleno==121); - case 222: /* between_op ::= BETWEEN */ yytestcase(yyruleno==222); - case 225: /* in_op ::= IN */ yytestcase(yyruleno==225); -{yygotominor.yy4 = 0;} + case 172: /* setlist ::= nm EQ expr */ +{ + yygotominor.yy322 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy118.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); +} break; - case 29: /* ifnotexists ::= IF NOT EXISTS */ - case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); - case 71: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==71); - case 86: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==86); - case 108: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==108); - case 119: /* distinct ::= DISTINCT */ yytestcase(yyruleno==119); - case 223: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==223); - case 226: /* in_op ::= NOT IN */ yytestcase(yyruleno==226); -{yygotominor.yy4 = 1;} + case 173: /* cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP */ +{sqlite3Insert(pParse, yymsp[-5].minor.yy259, yymsp[-1].minor.yy322, 0, yymsp[-4].minor.yy384, yymsp[-7].minor.yy210);} + break; + case 174: /* cmd ::= insert_cmd INTO fullname inscollist_opt select */ +{sqlite3Insert(pParse, yymsp[-2].minor.yy259, 0, yymsp[0].minor.yy387, yymsp[-1].minor.yy384, yymsp[-4].minor.yy210);} + break; + case 175: /* cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ +{sqlite3Insert(pParse, yymsp[-3].minor.yy259, 0, 0, yymsp[-2].minor.yy384, yymsp[-5].minor.yy210);} + break; + case 176: /* insert_cmd ::= INSERT orconf */ +{yygotominor.yy210 = yymsp[0].minor.yy210;} + break; + case 177: /* insert_cmd ::= REPLACE */ +{yygotominor.yy210 = OE_Replace;} + break; + case 178: /* itemlist ::= itemlist COMMA expr */ + case 241: /* nexprlist ::= nexprlist COMMA expr */ yytestcase(yyruleno==241); +{yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy118.pExpr);} + break; + case 179: /* itemlist ::= expr */ + case 242: /* nexprlist ::= expr */ yytestcase(yyruleno==242); +{yygotominor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy118.pExpr);} + break; + case 182: /* inscollist ::= inscollist COMMA nm */ +{yygotominor.yy384 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy384,&yymsp[0].minor.yy0);} + break; + case 183: /* inscollist ::= nm */ +{yygotominor.yy384 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} + break; + case 184: /* expr ::= term */ +{yygotominor.yy118 = yymsp[0].minor.yy118;} + break; + case 185: /* expr ::= LP expr RP */ +{yygotominor.yy118.pExpr = yymsp[-1].minor.yy118.pExpr; spanSet(&yygotominor.yy118,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 186: /* term ::= NULL */ + case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191); + case 192: /* term ::= STRING */ yytestcase(yyruleno==192); +{spanExpr(&yygotominor.yy118, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} + break; + case 187: /* expr ::= id */ + case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188); +{spanExpr(&yygotominor.yy118, pParse, TK_ID, &yymsp[0].minor.yy0);} + break; + case 189: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + spanSet(&yygotominor.yy118,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 190: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + spanSet(&yygotominor.yy118,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 193: /* expr ::= REGISTER */ +{ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); + yygotominor.yy118.pExpr = 0; + }else{ + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); + if( yygotominor.yy118.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy118.pExpr->iTable); + } + spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 194: /* expr ::= VARIABLE */ +{ + spanExpr(&yygotominor.yy118, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yygotominor.yy118.pExpr); + spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 195: /* expr ::= expr COLLATE ids */ +{ + yygotominor.yy118.pExpr = sqlite3ExprSetCollByToken(pParse, yymsp[-2].minor.yy118.pExpr, &yymsp[0].minor.yy0); + yygotominor.yy118.zStart = yymsp[-2].minor.yy118.zStart; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 196: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy118.pExpr, 0, &yymsp[-1].minor.yy0); + spanSet(&yygotominor.yy118,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 197: /* expr ::= ID LP distinct exprlist RP */ +{ + if( yymsp[-1].minor.yy322 && yymsp[-1].minor.yy322->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); + } + yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); + spanSet(&yygotominor.yy118,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy4 && yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->flags |= EP_Distinct; + } +} + break; + case 198: /* expr ::= ID LP STAR RP */ +{ + yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yygotominor.yy118,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); +} + break; + case 199: /* term ::= CTIME_KW */ +{ + /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are + ** treated as functions that return constants */ + yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, 0,&yymsp[0].minor.yy0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->op = TK_CONST_FUNC; + } + spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 200: /* expr ::= expr AND expr */ + case 201: /* expr ::= expr OR expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==204); + case 205: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==205); + case 206: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==206); + case 207: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==207); +{spanBinaryExpr(&yygotominor.yy118,pParse,yymsp[-1].major,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy118);} + break; + case 208: /* likeop ::= LIKE_KW */ + case 210: /* likeop ::= MATCH */ yytestcase(yyruleno==210); +{yygotominor.yy342.eOperator = yymsp[0].minor.yy0; yygotominor.yy342.not = 0;} + break; + case 209: /* likeop ::= NOT LIKE_KW */ + case 211: /* likeop ::= NOT MATCH */ yytestcase(yyruleno==211); +{yygotominor.yy342.eOperator = yymsp[0].minor.yy0; yygotominor.yy342.not = 1;} + break; + case 212: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy118.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy118.pExpr); + yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy342.eOperator); + if( yymsp[-1].minor.yy342.not ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); + yygotominor.yy118.zStart = yymsp[-2].minor.yy118.zStart; + yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; + if( yygotominor.yy118.pExpr ) yygotominor.yy118.pExpr->flags |= EP_InfixFunc; +} + break; + case 213: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy118.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy118.pExpr); + yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy342.eOperator); + if( yymsp[-3].minor.yy342.not ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); + yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; + yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; + if( yygotominor.yy118.pExpr ) yygotominor.yy118.pExpr->flags |= EP_InfixFunc; +} + break; + case 214: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(&yygotominor.yy118,pParse,yymsp[0].major,&yymsp[-1].minor.yy118,&yymsp[0].minor.yy0);} + break; + case 215: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(&yygotominor.yy118,pParse,TK_NOTNULL,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy0);} + break; + case 216: /* expr ::= expr IS expr */ +{ + spanBinaryExpr(&yygotominor.yy118,pParse,TK_IS,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy118); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy118.pExpr, yygotominor.yy118.pExpr, TK_ISNULL); +} + break; + case 217: /* expr ::= expr IS NOT expr */ +{ + spanBinaryExpr(&yygotominor.yy118,pParse,TK_ISNOT,&yymsp[-3].minor.yy118,&yymsp[0].minor.yy118); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy118.pExpr, yygotominor.yy118.pExpr, TK_NOTNULL); +} + break; + case 218: /* expr ::= NOT expr */ + case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219); +{spanUnaryPrefix(&yygotominor.yy118,pParse,yymsp[-1].major,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} + break; + case 220: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yygotominor.yy118,pParse,TK_UMINUS,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} + break; + case 221: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yygotominor.yy118,pParse,TK_UPLUS,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} + break; + case 224: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy118.pExpr); + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy118.pExpr, 0, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); + yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; + yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; +} + break; + case 227: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy322==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy4]); + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy118.pExpr); + }else{ + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy118.pExpr, 0, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->x.pList = yymsp[-1].minor.yy322; + sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + } + if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); + } + yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 228: /* expr ::= LP select RP */ +{ + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->x.pSelect = yymsp[-1].minor.yy387; + ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + } + yygotominor.yy118.zStart = yymsp[-2].minor.yy0.z; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 229: /* expr ::= expr in_op LP select RP */ +{ + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy118.pExpr, 0, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->x.pSelect = yymsp[-1].minor.yy387; + ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + } + if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); + yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } break; - case 32: /* create_table_args ::= LP columnlist conslist_opt RP */ + case 230: /* expr ::= expr in_op nm dbnm */ { - sqlite3EndTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0); -} + SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy118.pExpr, 0, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + }else{ + sqlite3SrcListDelete(pParse->db, pSrc); + } + if( yymsp[-2].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); + yygotominor.yy118.zStart = yymsp[-3].minor.yy118.zStart; + yygotominor.yy118.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + } break; - case 33: /* create_table_args ::= AS select */ + case 231: /* expr ::= EXISTS LP select RP */ { - sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy387); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy387); -} + Expr *p = yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + if( p ){ + p->x.pSelect = yymsp[-1].minor.yy387; + ExprSetProperty(p, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, p); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + } + yygotominor.yy118.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } break; - case 36: /* column ::= columnid type carglist */ + case 232: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yygotominor.yy0.z = yymsp[-2].minor.yy0.z; - yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy314, yymsp[-1].minor.yy314, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->x.pList = yymsp[-2].minor.yy322; + sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy322); + } + yygotominor.yy118.zStart = yymsp[-4].minor.yy0.z; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 37: /* columnid ::= nm */ + case 233: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); - yygotominor.yy0 = yymsp[0].minor.yy0; + yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy118.pExpr); + yygotominor.yy322 = sqlite3ExprListAppend(pParse,yygotominor.yy322, yymsp[0].minor.yy118.pExpr); } break; - case 38: /* id ::= ID */ - case 39: /* id ::= INDEXED */ yytestcase(yyruleno==39); - case 40: /* ids ::= ID|STRING */ yytestcase(yyruleno==40); - case 41: /* nm ::= id */ yytestcase(yyruleno==41); - case 42: /* nm ::= STRING */ yytestcase(yyruleno==42); - case 43: /* nm ::= JOIN_KW */ yytestcase(yyruleno==43); - case 46: /* typetoken ::= typename */ yytestcase(yyruleno==46); - case 49: /* typename ::= ids */ yytestcase(yyruleno==49); - case 127: /* as ::= AS nm */ yytestcase(yyruleno==127); - case 128: /* as ::= ids */ yytestcase(yyruleno==128); - case 138: /* dbnm ::= DOT nm */ yytestcase(yyruleno==138); - case 147: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==147); - case 251: /* collate ::= COLLATE ids */ yytestcase(yyruleno==251); - case 260: /* nmnum ::= plus_num */ yytestcase(yyruleno==260); - case 261: /* nmnum ::= nm */ yytestcase(yyruleno==261); - case 262: /* nmnum ::= ON */ yytestcase(yyruleno==262); - case 263: /* nmnum ::= DELETE */ yytestcase(yyruleno==263); - case 264: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==264); - case 265: /* plus_num ::= plus_opt number */ yytestcase(yyruleno==265); - case 266: /* minus_num ::= MINUS number */ yytestcase(yyruleno==266); - case 267: /* number ::= INTEGER|FLOAT */ yytestcase(yyruleno==267); - case 285: /* trnm ::= nm */ yytestcase(yyruleno==285); -{yygotominor.yy0 = yymsp[0].minor.yy0;} - break; - case 45: /* type ::= typetoken */ -{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} - break; - case 47: /* typetoken ::= typename LP signed RP */ + case 234: /* case_exprlist ::= WHEN expr THEN expr */ { - yygotominor.yy0.z = yymsp[-3].minor.yy0.z; - yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); + yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); + yygotominor.yy322 = sqlite3ExprListAppend(pParse,yygotominor.yy322, yymsp[0].minor.yy118.pExpr); } break; - case 48: /* typetoken ::= typename LP signed COMMA signed RP */ + case 243: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP */ { - yygotominor.yy0.z = yymsp[-5].minor.yy0.z; - yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); + sqlite3CreateIndex(pParse, &yymsp[-6].minor.yy0, &yymsp[-5].minor.yy0, + sqlite3SrcListAppend(pParse->db,0,&yymsp[-3].minor.yy0,0), yymsp[-1].minor.yy322, yymsp[-9].minor.yy4, + &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy4); } break; - case 50: /* typename ::= typename ids */ -{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} - break; - case 57: /* ccons ::= DEFAULT term */ - case 59: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==59); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy118);} + case 244: /* uniqueflag ::= UNIQUE */ + case 298: /* raisetype ::= ABORT */ yytestcase(yyruleno==298); +{yygotominor.yy4 = OE_Abort;} break; - case 58: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy118);} + case 245: /* uniqueflag ::= */ +{yygotominor.yy4 = OE_None;} break; - case 60: /* ccons ::= DEFAULT MINUS term */ + case 248: /* idxlist ::= idxlist COMMA nm collate sortorder */ { - ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy118.pExpr, 0, 0); - v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy118.zEnd; - sqlite3AddDefaultValue(pParse,&v); + Expr *p = 0; + if( yymsp[-1].minor.yy0.n>0 ){ + p = sqlite3Expr(pParse->db, TK_COLUMN, 0); + sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); + } + yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, p); + sqlite3ExprListSetName(pParse,yygotominor.yy322,&yymsp[-2].minor.yy0,1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy322, "index"); + if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; } break; - case 61: /* ccons ::= DEFAULT id */ + case 249: /* idxlist ::= nm collate sortorder */ { - ExprSpan v; - spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); - sqlite3AddDefaultValue(pParse,&v); + Expr *p = 0; + if( yymsp[-1].minor.yy0.n>0 ){ + p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); + } + yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, p); + sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy322, "index"); + if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; } break; - case 63: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy4);} - break; - case 64: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy4,yymsp[0].minor.yy4,yymsp[-2].minor.yy4);} - break; - case 65: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy4,0,0,0,0);} + case 250: /* collate ::= */ +{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} break; - case 66: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy118.pExpr);} + case 252: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy259, yymsp[-1].minor.yy4);} break; - case 67: /* ccons ::= REFERENCES nm idxlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy322,yymsp[0].minor.yy4);} + case 253: /* cmd ::= VACUUM */ + case 254: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==254); +{sqlite3Vacuum(pParse);} break; - case 68: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy4);} + case 255: /* cmd ::= PRAGMA nm dbnm */ +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 69: /* ccons ::= COLLATE ids */ -{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + case 256: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 72: /* refargs ::= */ -{ yygotominor.yy4 = OE_None*0x0101; /* EV: R-19803-45884 */} + case 257: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 73: /* refargs ::= refargs refarg */ -{ yygotominor.yy4 = (yymsp[-1].minor.yy4 & ~yymsp[0].minor.yy215.mask) | yymsp[0].minor.yy215.value; } + case 258: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 74: /* refarg ::= MATCH nm */ - case 75: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==75); -{ yygotominor.yy215.value = 0; yygotominor.yy215.mask = 0x000000; } + case 259: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 76: /* refarg ::= ON DELETE refact */ -{ yygotominor.yy215.value = yymsp[0].minor.yy4; yygotominor.yy215.mask = 0x0000ff; } + case 270: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ +{ + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy203, &all); +} break; - case 77: /* refarg ::= ON UPDATE refact */ -{ yygotominor.yy215.value = yymsp[0].minor.yy4<<8; yygotominor.yy215.mask = 0x00ff00; } + case 271: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy4, yymsp[-4].minor.yy90.a, yymsp[-4].minor.yy90.b, yymsp[-2].minor.yy259, yymsp[0].minor.yy314, yymsp[-10].minor.yy4, yymsp[-8].minor.yy4); + yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); +} break; - case 78: /* refact ::= SET NULL */ -{ yygotominor.yy4 = OE_SetNull; /* EV: R-33326-45252 */} + case 272: /* trigger_time ::= BEFORE */ + case 275: /* trigger_time ::= */ yytestcase(yyruleno==275); +{ yygotominor.yy4 = TK_BEFORE; } break; - case 79: /* refact ::= SET DEFAULT */ -{ yygotominor.yy4 = OE_SetDflt; /* EV: R-33326-45252 */} + case 273: /* trigger_time ::= AFTER */ +{ yygotominor.yy4 = TK_AFTER; } break; - case 80: /* refact ::= CASCADE */ -{ yygotominor.yy4 = OE_Cascade; /* EV: R-33326-45252 */} + case 274: /* trigger_time ::= INSTEAD OF */ +{ yygotominor.yy4 = TK_INSTEAD;} break; - case 81: /* refact ::= RESTRICT */ -{ yygotominor.yy4 = OE_Restrict; /* EV: R-33326-45252 */} + case 276: /* trigger_event ::= DELETE|INSERT */ + case 277: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==277); +{yygotominor.yy90.a = yymsp[0].major; yygotominor.yy90.b = 0;} break; - case 82: /* refact ::= NO ACTION */ -{ yygotominor.yy4 = OE_None; /* EV: R-33326-45252 */} + case 278: /* trigger_event ::= UPDATE OF inscollist */ +{yygotominor.yy90.a = TK_UPDATE; yygotominor.yy90.b = yymsp[0].minor.yy384;} break; - case 84: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 99: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==99); - case 101: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==101); - case 104: /* resolvetype ::= raisetype */ yytestcase(yyruleno==104); -{yygotominor.yy4 = yymsp[0].minor.yy4;} + case 281: /* when_clause ::= */ + case 303: /* key_opt ::= */ yytestcase(yyruleno==303); +{ yygotominor.yy314 = 0; } break; - case 88: /* conslist_opt ::= */ -{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} + case 282: /* when_clause ::= WHEN expr */ + case 304: /* key_opt ::= KEY expr */ yytestcase(yyruleno==304); +{ yygotominor.yy314 = yymsp[0].minor.yy118.pExpr; } break; - case 89: /* conslist_opt ::= COMMA conslist */ -{yygotominor.yy0 = yymsp[-1].minor.yy0;} + case 283: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy203!=0 ); + yymsp[-2].minor.yy203->pLast->pNext = yymsp[-1].minor.yy203; + yymsp[-2].minor.yy203->pLast = yymsp[-1].minor.yy203; + yygotominor.yy203 = yymsp[-2].minor.yy203; +} break; - case 94: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy322,yymsp[0].minor.yy4,yymsp[-2].minor.yy4,0);} + case 284: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy203!=0 ); + yymsp[-1].minor.yy203->pLast = yymsp[-1].minor.yy203; + yygotominor.yy203 = yymsp[-1].minor.yy203; +} break; - case 95: /* tcons ::= UNIQUE LP idxlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy322,yymsp[0].minor.yy4,0,0,0,0);} + case 286: /* trnm ::= nm DOT nm */ +{ + yygotominor.yy0 = yymsp[0].minor.yy0; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} break; - case 96: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy118.pExpr);} + case 288: /* tridxby ::= INDEXED BY nm */ +{ + sqlite3ErrorMsg(pParse, + "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} break; - case 97: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ + case 289: /* tridxby ::= NOT INDEXED */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy322, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[-1].minor.yy4); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy4); + sqlite3ErrorMsg(pParse, + "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " + "within triggers"); } break; - case 100: /* onconf ::= */ -{yygotominor.yy4 = OE_Default;} + case 290: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{ yygotominor.yy203 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy322, yymsp[0].minor.yy314, yymsp[-5].minor.yy210); } break; - case 102: /* orconf ::= */ -{yygotominor.yy210 = OE_Default;} + case 291: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP */ +{yygotominor.yy203 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy384, yymsp[-1].minor.yy322, 0, yymsp[-7].minor.yy210);} break; - case 103: /* orconf ::= OR resolvetype */ -{yygotominor.yy210 = (u8)yymsp[0].minor.yy4;} + case 292: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ +{yygotominor.yy203 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy384, 0, yymsp[0].minor.yy387, yymsp[-4].minor.yy210);} break; - case 105: /* resolvetype ::= IGNORE */ -{yygotominor.yy4 = OE_Ignore;} + case 293: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yygotominor.yy203 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy314);} break; - case 106: /* resolvetype ::= REPLACE */ -{yygotominor.yy4 = OE_Replace;} + case 294: /* trigger_cmd ::= select */ +{yygotominor.yy203 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy387); } break; - case 107: /* cmd ::= DROP TABLE ifexists fullname */ + case 295: /* expr ::= RAISE LP IGNORE RP */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy259, 0, yymsp[-1].minor.yy4); + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yygotominor.yy118.pExpr ){ + yygotominor.yy118.pExpr->affinity = OE_Ignore; + } + yygotominor.yy118.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 110: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ + case 296: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy387, yymsp[-6].minor.yy4, yymsp[-4].minor.yy4); + yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yygotominor.yy118.pExpr ) { + yygotominor.yy118.pExpr->affinity = (char)yymsp[-3].minor.yy4; + } + yygotominor.yy118.zStart = yymsp[-5].minor.yy0.z; + yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 111: /* cmd ::= DROP VIEW ifexists fullname */ + case 297: /* raisetype ::= ROLLBACK */ +{yygotominor.yy4 = OE_Rollback;} + break; + case 299: /* raisetype ::= FAIL */ +{yygotominor.yy4 = OE_Fail;} + break; + case 300: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy259, 1, yymsp[-1].minor.yy4); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy259,yymsp[-1].minor.yy4); } break; - case 112: /* cmd ::= select */ + case 301: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - SelectDest dest = {SRT_Output, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy387, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy387); + sqlite3Attach(pParse, yymsp[-3].minor.yy118.pExpr, yymsp[-1].minor.yy118.pExpr, yymsp[0].minor.yy314); } break; - case 113: /* select ::= oneselect */ -{yygotominor.yy387 = yymsp[0].minor.yy387;} - break; - case 114: /* select ::= select multiselect_op oneselect */ + case 302: /* cmd ::= DETACH database_kw_opt expr */ { - if( yymsp[0].minor.yy387 ){ - yymsp[0].minor.yy387->op = (u8)yymsp[-1].minor.yy4; - yymsp[0].minor.yy387->pPrior = yymsp[-2].minor.yy387; - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy387); - } - yygotominor.yy387 = yymsp[0].minor.yy387; + sqlite3Detach(pParse, yymsp[0].minor.yy118.pExpr); } break; - case 116: /* multiselect_op ::= UNION ALL */ -{yygotominor.yy4 = TK_ALL;} + case 307: /* cmd ::= REINDEX */ +{sqlite3Reindex(pParse, 0, 0);} break; - case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -{ - yygotominor.yy387 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy322,yymsp[-5].minor.yy259,yymsp[-4].minor.yy314,yymsp[-3].minor.yy322,yymsp[-2].minor.yy314,yymsp[-1].minor.yy322,yymsp[-7].minor.yy4,yymsp[0].minor.yy292.pLimit,yymsp[0].minor.yy292.pOffset); -} + case 308: /* cmd ::= REINDEX nm dbnm */ +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 122: /* sclp ::= selcollist COMMA */ - case 247: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==247); -{yygotominor.yy322 = yymsp[-1].minor.yy322;} + case 309: /* cmd ::= ANALYZE */ +{sqlite3Analyze(pParse, 0, 0);} break; - case 123: /* sclp ::= */ - case 151: /* orderby_opt ::= */ yytestcase(yyruleno==151); - case 159: /* groupby_opt ::= */ yytestcase(yyruleno==159); - case 240: /* exprlist ::= */ yytestcase(yyruleno==240); - case 246: /* idxlist_opt ::= */ yytestcase(yyruleno==246); -{yygotominor.yy322 = 0;} + case 310: /* cmd ::= ANALYZE nm dbnm */ +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 124: /* selcollist ::= sclp expr as */ + case 311: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, yymsp[-1].minor.yy118.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yygotominor.yy322,&yymsp[-1].minor.yy118); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy259,&yymsp[0].minor.yy0); } break; - case 125: /* selcollist ::= sclp STAR */ + case 312: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ { - Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); - yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy322, p); + sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); } break; - case 126: /* selcollist ::= sclp nm DOT STAR */ + case 313: /* add_column_fullname ::= fullname */ { - Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); - Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, pDot); + pParse->db->lookaside.bEnabled = 0; + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy259); } break; - case 129: /* as ::= */ -{yygotominor.yy0.n = 0;} + case 316: /* cmd ::= create_vtab */ +{sqlite3VtabFinishParse(pParse,0);} break; - case 130: /* from ::= */ -{yygotominor.yy259 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy259));} + case 317: /* cmd ::= create_vtab LP vtabarglist RP */ +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 131: /* from ::= FROM seltablist */ + case 318: /* create_vtab ::= createkw VIRTUAL TABLE nm dbnm USING nm */ { - yygotominor.yy259 = yymsp[0].minor.yy259; - sqlite3SrcListShiftJoinType(yygotominor.yy259); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 132: /* stl_prefix ::= seltablist joinop */ -{ - yygotominor.yy259 = yymsp[-1].minor.yy259; - if( ALWAYS(yygotominor.yy259 && yygotominor.yy259->nSrc>0) ) yygotominor.yy259->a[yygotominor.yy259->nSrc-1].jointype = (u8)yymsp[0].minor.yy4; -} + case 321: /* vtabarg ::= */ +{sqlite3VtabArgInit(pParse);} break; - case 133: /* stl_prefix ::= */ -{yygotominor.yy259 = 0;} + case 323: /* vtabargtoken ::= ANY */ + case 324: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==324); + case 325: /* lp ::= LP */ yytestcase(yyruleno==325); +{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 134: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ -{ - yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); - sqlite3SrcListIndexedBy(pParse, yygotominor.yy259, &yymsp[-2].minor.yy0); + default: + /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); + /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); + /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2); + /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3); + /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4); + /* (10) trans_opt ::= */ yytestcase(yyruleno==10); + /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11); + /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12); + /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); + /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); + /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); + /* (34) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==34); + /* (35) columnlist ::= column */ yytestcase(yyruleno==35); + /* (44) type ::= */ yytestcase(yyruleno==44); + /* (51) signed ::= plus_num */ yytestcase(yyruleno==51); + /* (52) signed ::= minus_num */ yytestcase(yyruleno==52); + /* (53) carglist ::= carglist carg */ yytestcase(yyruleno==53); + /* (54) carglist ::= */ yytestcase(yyruleno==54); + /* (55) carg ::= CONSTRAINT nm ccons */ yytestcase(yyruleno==55); + /* (56) carg ::= ccons */ yytestcase(yyruleno==56); + /* (62) ccons ::= NULL onconf */ yytestcase(yyruleno==62); + /* (90) conslist ::= conslist COMMA tcons */ yytestcase(yyruleno==90); + /* (91) conslist ::= conslist tcons */ yytestcase(yyruleno==91); + /* (92) conslist ::= tcons */ yytestcase(yyruleno==92); + /* (93) tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==93); + /* (268) plus_opt ::= PLUS */ yytestcase(yyruleno==268); + /* (269) plus_opt ::= */ yytestcase(yyruleno==269); + /* (279) foreach_clause ::= */ yytestcase(yyruleno==279); + /* (280) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==280); + /* (287) tridxby ::= */ yytestcase(yyruleno==287); + /* (305) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==305); + /* (306) database_kw_opt ::= */ yytestcase(yyruleno==306); + /* (314) kwcolumn_opt ::= */ yytestcase(yyruleno==314); + /* (315) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==315); + /* (319) vtabarglist ::= vtabarg */ yytestcase(yyruleno==319); + /* (320) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==320); + /* (322) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==322); + /* (326) anylist ::= */ yytestcase(yyruleno==326); + /* (327) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==327); + /* (328) anylist ::= anylist ANY */ yytestcase(yyruleno==328); + break; + }; + yygoto = yyRuleInfo[yyruleno].lhs; + yysize = yyRuleInfo[yyruleno].nrhs; + yypParser->yyidx -= yysize; + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); + if( yyact < YYNSTATE ){ +#ifdef NDEBUG + /* If we are not debugging and the reduce action popped at least + ** one element off the stack, then we can push the new element back + ** onto the stack here, and skip the stack overflow test in yy_shift(). + ** That gives a significant speed improvement. */ + if( yysize ){ + yypParser->yyidx++; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yymsp->minor = yygotominor; + }else +#endif + { + yy_shift(yypParser,yyact,yygoto,&yygotominor); + } + }else{ + assert( yyact == YYNSTATE + YYNRULE + 1 ); + yy_accept(yypParser); + } +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + YYMINORTYPE yyminor /* The minor type of the error token */ +){ + sqlite3ParserARG_FETCH; +#define TOKEN (yyminor.yy0) + + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + pParse->parseError = 1; + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } - break; - case 135: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ -{ - yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy387,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } - break; - case 136: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ -{ - if( yymsp[-6].minor.yy259==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy314==0 && yymsp[0].minor.yy384==0 ){ - yygotominor.yy259 = yymsp[-4].minor.yy259; +#endif + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ + sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
      +**
    • A pointer to the parser (an opaque structure.) +**
    • The major token number. +**
    • The minor token number. +**
    • An option argument of a grammar-specified type. +**
    +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3Parser( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + sqlite3ParserTOKENTYPE yyminor /* The value for the token */ + sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + int yyact; /* The parser action. */ + int yyendofinput; /* True if we are at the end of input */ +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser; /* The parser */ + + /* (re)initialize the parser, if necessary */ + yypParser = (yyParser*)yyp; + if( yypParser->yyidx<0 ){ +#if YYSTACKDEPTH<=0 + if( yypParser->yystksz <=0 ){ + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; + yyStackOverflow(yypParser, &yyminorunion); + return; + } +#endif + yypParser->yyidx = 0; + yypParser->yyerrcnt = -1; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; + } + yyminorunion.yy0 = yyminor; + yyendofinput = (yymajor==0); + sqlite3ParserARG_STORE; + +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); + } +#endif + + do{ + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + if( yyactyyerrcnt--; + yymajor = YYNOCODE; + }else if( yyact < YYNSTATE + YYNRULE ){ + yy_reduce(yypParser,yyact-YYNSTATE); }else{ - Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy259); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy259,0,0,0,0,0,0,0); - yygotominor.yy259 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy259,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy314,yymsp[0].minor.yy384); + assert( yyact == YY_ERROR_ACTION ); +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminorunion); + } + yymx = yypParser->yystack[yypParser->yyidx].major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + }else{ + while( + yypParser->yyidx >= 0 && + yymx != YYERRORSYMBOL && + (yyact = yy_find_reduce_action( + yypParser->yystack[yypParser->yyidx].stateno, + YYERRORSYMBOL)) >= YYNSTATE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yyidx < 0 || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + YYMINORTYPE u2; + u2.YYERRSYMDT = 0; + yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor,yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yymajor = YYNOCODE; + +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor,yyminorunion); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); + } + yymajor = YYNOCODE; +#endif + } + }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); + return; +} + +/************** End of parse.c ***********************************************/ +/************** Begin file tokenize.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that splits an SQL input string up into +** individual tokens and sends those tokens one-by-one over to the +** parser for analysis. +*/ + +/* +** The charMap() macro maps alphabetic characters into their +** lower-case ASCII equivalent. On ASCII machines, this is just +** an upper-to-lower case map. On EBCDIC machines we also need +** to adjust the encoding. Only alphabetic characters and underscores +** need to be translated. +*/ +#ifdef SQLITE_ASCII +# define charMap(X) sqlite3UpperToLower[(unsigned char)X] +#endif +#ifdef SQLITE_EBCDIC +# define charMap(X) ebcdicToAscii[(unsigned char)X] +const unsigned char ebcdicToAscii[] = { +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ +}; +#endif + +/* +** The sqlite3KeywordCode function looks up an identifier to determine if +** it is a keyword. If it is a keyword, the token code of that keyword is +** returned. If the input is not a keyword, TK_ID is returned. +** +** The implementation of this routine was generated by a program, +** mkkeywordhash.h, located in the tool subdirectory of the distribution. +** The output of the mkkeywordhash.c program is written into a file +** named keywordhash.h and then included into this source file by +** the #include below. +*/ +/************** Include keywordhash.h in the middle of tokenize.c ************/ +/************** Begin file keywordhash.h *************************************/ +/***** This file contains automatically generated code ****** +** +** The code in this file has been automatically generated by +** +** sqlite/tool/mkkeywordhash.c +** +** The code in this file implements a function that determines whether +** or not a given identifier is really an SQL keyword. The same thing +** might be implemented more directly using a hand-written hash table. +** But by using this automatically generated code, the size of the code +** is substantially reduced. This is important for embedded applications +** on platforms with limited memory. +*/ +/* Hash score: 175 */ +static int keywordCode(const char *z, int n){ + /* zText[] encodes 811 bytes of keywords in 541 bytes */ + /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ + /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ + /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ + /* UNIQUERYATTACHAVINGROUPDATEBEGINNERELEASEBETWEENOTNULLIKE */ + /* CASCADELETECASECOLLATECREATECURRENT_DATEDETACHIMMEDIATEJOIN */ + /* SERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHENWHERENAME */ + /* AFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSS */ + /* CURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAILFROMFULLGLOBYIF */ + /* ISNULLORDERESTRICTOUTERIGHTROLLBACKROWUNIONUSINGVACUUMVIEW */ + /* INITIALLY */ + static const char zText[540] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', + 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', + 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', + 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', + 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', + 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', + 'U','E','R','Y','A','T','T','A','C','H','A','V','I','N','G','R','O','U', + 'P','D','A','T','E','B','E','G','I','N','N','E','R','E','L','E','A','S', + 'E','B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C', + 'A','S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L', + 'A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D', + 'A','T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E', + 'J','O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A', + 'L','Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U', + 'E','S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W', + 'H','E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C', + 'E','A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R', + 'E','M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M', + 'M','I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U', + 'R','R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M', + 'A','R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T', + 'D','R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L', + 'O','B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S', + 'T','R','I','C','T','O','U','T','E','R','I','G','H','T','R','O','L','L', + 'B','A','C','K','R','O','W','U','N','I','O','N','U','S','I','N','G','V', + 'A','C','U','U','M','V','I','E','W','I','N','I','T','I','A','L','L','Y', + }; + static const unsigned char aHash[127] = { + 72, 101, 114, 70, 0, 45, 0, 0, 78, 0, 73, 0, 0, + 42, 12, 74, 15, 0, 113, 81, 50, 108, 0, 19, 0, 0, + 118, 0, 116, 111, 0, 22, 89, 0, 9, 0, 0, 66, 67, + 0, 65, 6, 0, 48, 86, 98, 0, 115, 97, 0, 0, 44, + 0, 99, 24, 0, 17, 0, 119, 49, 23, 0, 5, 106, 25, + 92, 0, 0, 121, 102, 56, 120, 53, 28, 51, 0, 87, 0, + 96, 26, 0, 95, 0, 0, 0, 91, 88, 93, 84, 105, 14, + 39, 104, 0, 77, 0, 18, 85, 107, 32, 0, 117, 76, 109, + 58, 46, 80, 0, 0, 90, 40, 0, 112, 0, 36, 0, 0, + 29, 0, 82, 59, 60, 0, 20, 57, 0, 52, + }; + static const unsigned char aNext[121] = { + 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, + 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 43, 3, 47, + 0, 0, 0, 0, 30, 0, 54, 0, 38, 0, 0, 0, 1, + 62, 0, 0, 63, 0, 41, 0, 0, 0, 0, 0, 0, 0, + 61, 0, 0, 0, 0, 31, 55, 16, 34, 10, 0, 0, 0, + 0, 0, 0, 0, 11, 68, 75, 0, 8, 0, 100, 94, 0, + 103, 0, 83, 0, 71, 0, 0, 110, 27, 37, 69, 79, 0, + 35, 64, 0, 0, + }; + static const unsigned char aLen[121] = { + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, + 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, + 4, 6, 2, 3, 9, 4, 2, 6, 5, 6, 6, 5, 6, + 5, 5, 7, 7, 7, 3, 2, 4, 4, 7, 3, 6, 4, + 7, 6, 12, 6, 9, 4, 6, 5, 4, 7, 6, 5, 6, + 7, 5, 4, 5, 6, 5, 7, 3, 7, 13, 2, 2, 4, + 6, 6, 8, 5, 17, 12, 7, 8, 8, 2, 4, 4, 4, + 4, 4, 2, 2, 6, 5, 8, 5, 5, 8, 3, 5, 5, + 6, 4, 9, 3, + }; + static const unsigned short int aOffset[121] = { + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, + 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 189, 194, 197, + 203, 206, 210, 217, 223, 223, 223, 226, 229, 233, 234, 238, 244, + 248, 255, 261, 273, 279, 288, 290, 296, 301, 303, 310, 315, 320, + 326, 332, 337, 341, 344, 350, 354, 361, 363, 370, 372, 374, 383, + 387, 393, 399, 407, 412, 412, 428, 435, 442, 443, 450, 454, 458, + 462, 466, 469, 471, 473, 479, 483, 491, 495, 500, 508, 511, 516, + 521, 527, 531, 536, + }; + static const unsigned char aCode[121] = { + TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, + TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, + TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, + TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, + TK_OR, TK_UNIQUE, TK_QUERY, TK_ATTACH, TK_HAVING, + TK_GROUP, TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RELEASE, + TK_BETWEEN, TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, + TK_LIKE_KW, TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, + TK_COLLATE, TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, + TK_JOIN, TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, + TK_PRAGMA, TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, + TK_WHEN, TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, + TK_AND, TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, + TK_CAST, TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, + TK_CTIME_KW, TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, + TK_IS, TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, + TK_LIKE_KW, TK_BY, TK_IF, TK_ISNULL, TK_ORDER, + TK_RESTRICT, TK_JOIN_KW, TK_JOIN_KW, TK_ROLLBACK, TK_ROW, + TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, + TK_ALL, + }; + int h, i; + if( n<2 ) return TK_ID; + h = ((charMap(z[0])*4) ^ + (charMap(z[n-1])*3) ^ + n) % 127; + for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ + if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ + testcase( i==0 ); /* REINDEX */ + testcase( i==1 ); /* INDEXED */ + testcase( i==2 ); /* INDEX */ + testcase( i==3 ); /* DESC */ + testcase( i==4 ); /* ESCAPE */ + testcase( i==5 ); /* EACH */ + testcase( i==6 ); /* CHECK */ + testcase( i==7 ); /* KEY */ + testcase( i==8 ); /* BEFORE */ + testcase( i==9 ); /* FOREIGN */ + testcase( i==10 ); /* FOR */ + testcase( i==11 ); /* IGNORE */ + testcase( i==12 ); /* REGEXP */ + testcase( i==13 ); /* EXPLAIN */ + testcase( i==14 ); /* INSTEAD */ + testcase( i==15 ); /* ADD */ + testcase( i==16 ); /* DATABASE */ + testcase( i==17 ); /* AS */ + testcase( i==18 ); /* SELECT */ + testcase( i==19 ); /* TABLE */ + testcase( i==20 ); /* LEFT */ + testcase( i==21 ); /* THEN */ + testcase( i==22 ); /* END */ + testcase( i==23 ); /* DEFERRABLE */ + testcase( i==24 ); /* ELSE */ + testcase( i==25 ); /* EXCEPT */ + testcase( i==26 ); /* TRANSACTION */ + testcase( i==27 ); /* ACTION */ + testcase( i==28 ); /* ON */ + testcase( i==29 ); /* NATURAL */ + testcase( i==30 ); /* ALTER */ + testcase( i==31 ); /* RAISE */ + testcase( i==32 ); /* EXCLUSIVE */ + testcase( i==33 ); /* EXISTS */ + testcase( i==34 ); /* SAVEPOINT */ + testcase( i==35 ); /* INTERSECT */ + testcase( i==36 ); /* TRIGGER */ + testcase( i==37 ); /* REFERENCES */ + testcase( i==38 ); /* CONSTRAINT */ + testcase( i==39 ); /* INTO */ + testcase( i==40 ); /* OFFSET */ + testcase( i==41 ); /* OF */ + testcase( i==42 ); /* SET */ + testcase( i==43 ); /* TEMPORARY */ + testcase( i==44 ); /* TEMP */ + testcase( i==45 ); /* OR */ + testcase( i==46 ); /* UNIQUE */ + testcase( i==47 ); /* QUERY */ + testcase( i==48 ); /* ATTACH */ + testcase( i==49 ); /* HAVING */ + testcase( i==50 ); /* GROUP */ + testcase( i==51 ); /* UPDATE */ + testcase( i==52 ); /* BEGIN */ + testcase( i==53 ); /* INNER */ + testcase( i==54 ); /* RELEASE */ + testcase( i==55 ); /* BETWEEN */ + testcase( i==56 ); /* NOTNULL */ + testcase( i==57 ); /* NOT */ + testcase( i==58 ); /* NO */ + testcase( i==59 ); /* NULL */ + testcase( i==60 ); /* LIKE */ + testcase( i==61 ); /* CASCADE */ + testcase( i==62 ); /* ASC */ + testcase( i==63 ); /* DELETE */ + testcase( i==64 ); /* CASE */ + testcase( i==65 ); /* COLLATE */ + testcase( i==66 ); /* CREATE */ + testcase( i==67 ); /* CURRENT_DATE */ + testcase( i==68 ); /* DETACH */ + testcase( i==69 ); /* IMMEDIATE */ + testcase( i==70 ); /* JOIN */ + testcase( i==71 ); /* INSERT */ + testcase( i==72 ); /* MATCH */ + testcase( i==73 ); /* PLAN */ + testcase( i==74 ); /* ANALYZE */ + testcase( i==75 ); /* PRAGMA */ + testcase( i==76 ); /* ABORT */ + testcase( i==77 ); /* VALUES */ + testcase( i==78 ); /* VIRTUAL */ + testcase( i==79 ); /* LIMIT */ + testcase( i==80 ); /* WHEN */ + testcase( i==81 ); /* WHERE */ + testcase( i==82 ); /* RENAME */ + testcase( i==83 ); /* AFTER */ + testcase( i==84 ); /* REPLACE */ + testcase( i==85 ); /* AND */ + testcase( i==86 ); /* DEFAULT */ + testcase( i==87 ); /* AUTOINCREMENT */ + testcase( i==88 ); /* TO */ + testcase( i==89 ); /* IN */ + testcase( i==90 ); /* CAST */ + testcase( i==91 ); /* COLUMN */ + testcase( i==92 ); /* COMMIT */ + testcase( i==93 ); /* CONFLICT */ + testcase( i==94 ); /* CROSS */ + testcase( i==95 ); /* CURRENT_TIMESTAMP */ + testcase( i==96 ); /* CURRENT_TIME */ + testcase( i==97 ); /* PRIMARY */ + testcase( i==98 ); /* DEFERRED */ + testcase( i==99 ); /* DISTINCT */ + testcase( i==100 ); /* IS */ + testcase( i==101 ); /* DROP */ + testcase( i==102 ); /* FAIL */ + testcase( i==103 ); /* FROM */ + testcase( i==104 ); /* FULL */ + testcase( i==105 ); /* GLOB */ + testcase( i==106 ); /* BY */ + testcase( i==107 ); /* IF */ + testcase( i==108 ); /* ISNULL */ + testcase( i==109 ); /* ORDER */ + testcase( i==110 ); /* RESTRICT */ + testcase( i==111 ); /* OUTER */ + testcase( i==112 ); /* RIGHT */ + testcase( i==113 ); /* ROLLBACK */ + testcase( i==114 ); /* ROW */ + testcase( i==115 ); /* UNION */ + testcase( i==116 ); /* USING */ + testcase( i==117 ); /* VACUUM */ + testcase( i==118 ); /* VIEW */ + testcase( i==119 ); /* INITIALLY */ + testcase( i==120 ); /* ALL */ + return aCode[i]; } } - break; - case 137: /* dbnm ::= */ - case 146: /* indexed_opt ::= */ yytestcase(yyruleno==146); -{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} - break; - case 139: /* fullname ::= nm dbnm */ -{yygotominor.yy259 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} - break; - case 140: /* joinop ::= COMMA|JOIN */ -{ yygotominor.yy4 = JT_INNER; } - break; - case 141: /* joinop ::= JOIN_KW JOIN */ -{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } - break; - case 142: /* joinop ::= JOIN_KW nm JOIN */ -{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } - break; - case 143: /* joinop ::= JOIN_KW nm nm JOIN */ -{ yygotominor.yy4 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } - break; - case 144: /* on_opt ::= ON expr */ - case 155: /* sortitem ::= expr */ yytestcase(yyruleno==155); - case 162: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==162); - case 169: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==169); - case 235: /* case_else ::= ELSE expr */ yytestcase(yyruleno==235); - case 237: /* case_operand ::= expr */ yytestcase(yyruleno==237); -{yygotominor.yy314 = yymsp[0].minor.yy118.pExpr;} - break; - case 145: /* on_opt ::= */ - case 161: /* having_opt ::= */ yytestcase(yyruleno==161); - case 168: /* where_opt ::= */ yytestcase(yyruleno==168); - case 236: /* case_else ::= */ yytestcase(yyruleno==236); - case 238: /* case_operand ::= */ yytestcase(yyruleno==238); -{yygotominor.yy314 = 0;} - break; - case 148: /* indexed_opt ::= NOT INDEXED */ -{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} - break; - case 149: /* using_opt ::= USING LP inscollist RP */ - case 181: /* inscollist_opt ::= LP inscollist RP */ yytestcase(yyruleno==181); -{yygotominor.yy384 = yymsp[-1].minor.yy384;} - break; - case 150: /* using_opt ::= */ - case 180: /* inscollist_opt ::= */ yytestcase(yyruleno==180); -{yygotominor.yy384 = 0;} - break; - case 152: /* orderby_opt ::= ORDER BY sortlist */ - case 160: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==160); - case 239: /* exprlist ::= nexprlist */ yytestcase(yyruleno==239); -{yygotominor.yy322 = yymsp[0].minor.yy322;} - break; - case 153: /* sortlist ::= sortlist COMMA sortitem sortorder */ -{ - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322,yymsp[-1].minor.yy314); - if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; -} - break; - case 154: /* sortlist ::= sortitem sortorder */ -{ - yygotominor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy314); - if( yygotominor.yy322 && ALWAYS(yygotominor.yy322->a) ) yygotominor.yy322->a[0].sortOrder = (u8)yymsp[0].minor.yy4; -} - break; - case 156: /* sortorder ::= ASC */ - case 158: /* sortorder ::= */ yytestcase(yyruleno==158); -{yygotominor.yy4 = SQLITE_SO_ASC;} - break; - case 157: /* sortorder ::= DESC */ -{yygotominor.yy4 = SQLITE_SO_DESC;} - break; - case 163: /* limit_opt ::= */ -{yygotominor.yy292.pLimit = 0; yygotominor.yy292.pOffset = 0;} - break; - case 164: /* limit_opt ::= LIMIT expr */ -{yygotominor.yy292.pLimit = yymsp[0].minor.yy118.pExpr; yygotominor.yy292.pOffset = 0;} - break; - case 165: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yygotominor.yy292.pLimit = yymsp[-2].minor.yy118.pExpr; yygotominor.yy292.pOffset = yymsp[0].minor.yy118.pExpr;} - break; - case 166: /* limit_opt ::= LIMIT expr COMMA expr */ -{yygotominor.yy292.pOffset = yymsp[-2].minor.yy118.pExpr; yygotominor.yy292.pLimit = yymsp[0].minor.yy118.pExpr;} - break; - case 167: /* cmd ::= DELETE FROM fullname indexed_opt where_opt */ -{ - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy259, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy259,yymsp[0].minor.yy314); -} - break; - case 170: /* cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt */ -{ - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy259, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy322,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy259,yymsp[-1].minor.yy322,yymsp[0].minor.yy314,yymsp[-5].minor.yy210); -} - break; - case 171: /* setlist ::= setlist COMMA nm EQ expr */ -{ - yygotominor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[0].minor.yy118.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); -} - break; - case 172: /* setlist ::= nm EQ expr */ -{ - yygotominor.yy322 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy118.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); -} - break; - case 173: /* cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP */ -{sqlite3Insert(pParse, yymsp[-5].minor.yy259, yymsp[-1].minor.yy322, 0, yymsp[-4].minor.yy384, yymsp[-7].minor.yy210);} - break; - case 174: /* cmd ::= insert_cmd INTO fullname inscollist_opt select */ -{sqlite3Insert(pParse, yymsp[-2].minor.yy259, 0, yymsp[0].minor.yy387, yymsp[-1].minor.yy384, yymsp[-4].minor.yy210);} - break; - case 175: /* cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ -{sqlite3Insert(pParse, yymsp[-3].minor.yy259, 0, 0, yymsp[-2].minor.yy384, yymsp[-5].minor.yy210);} - break; - case 176: /* insert_cmd ::= INSERT orconf */ -{yygotominor.yy210 = yymsp[0].minor.yy210;} - break; - case 177: /* insert_cmd ::= REPLACE */ -{yygotominor.yy210 = OE_Replace;} - break; - case 178: /* itemlist ::= itemlist COMMA expr */ - case 241: /* nexprlist ::= nexprlist COMMA expr */ yytestcase(yyruleno==241); -{yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy118.pExpr);} - break; - case 179: /* itemlist ::= expr */ - case 242: /* nexprlist ::= expr */ yytestcase(yyruleno==242); -{yygotominor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy118.pExpr);} - break; - case 182: /* inscollist ::= inscollist COMMA nm */ -{yygotominor.yy384 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy384,&yymsp[0].minor.yy0);} - break; - case 183: /* inscollist ::= nm */ -{yygotominor.yy384 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} - break; - case 184: /* expr ::= term */ -{yygotominor.yy118 = yymsp[0].minor.yy118;} - break; - case 185: /* expr ::= LP expr RP */ -{yygotominor.yy118.pExpr = yymsp[-1].minor.yy118.pExpr; spanSet(&yygotominor.yy118,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} - break; - case 186: /* term ::= NULL */ - case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191); - case 192: /* term ::= STRING */ yytestcase(yyruleno==192); -{spanExpr(&yygotominor.yy118, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} - break; - case 187: /* expr ::= id */ - case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188); -{spanExpr(&yygotominor.yy118, pParse, TK_ID, &yymsp[0].minor.yy0);} - break; - case 189: /* expr ::= nm DOT nm */ -{ - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&yygotominor.yy118,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); -} - break; - case 190: /* expr ::= nm DOT nm DOT nm */ -{ - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&yygotominor.yy118,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); -} - break; - case 193: /* expr ::= REGISTER */ -{ - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); - yygotominor.yy118.pExpr = 0; - }else{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); - if( yygotominor.yy118.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy118.pExpr->iTable); - } - spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); -} - break; - case 194: /* expr ::= VARIABLE */ -{ - spanExpr(&yygotominor.yy118, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yygotominor.yy118.pExpr); - spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); -} - break; - case 195: /* expr ::= expr COLLATE ids */ -{ - yygotominor.yy118.pExpr = sqlite3ExprSetCollByToken(pParse, yymsp[-2].minor.yy118.pExpr, &yymsp[0].minor.yy0); - yygotominor.yy118.zStart = yymsp[-2].minor.yy118.zStart; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 196: /* expr ::= CAST LP expr AS typetoken RP */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy118.pExpr, 0, &yymsp[-1].minor.yy0); - spanSet(&yygotominor.yy118,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); -} - break; - case 197: /* expr ::= ID LP distinct exprlist RP */ -{ - if( yymsp[-1].minor.yy322 && yymsp[-1].minor.yy322->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); - } - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); - spanSet(&yygotominor.yy118,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy4 && yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->flags |= EP_Distinct; - } -} - break; - case 198: /* expr ::= ID LP STAR RP */ -{ - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yygotominor.yy118,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); -} - break; - case 199: /* term ::= CTIME_KW */ -{ - /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are - ** treated as functions that return constants */ - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, 0,&yymsp[0].minor.yy0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->op = TK_CONST_FUNC; - } - spanSet(&yygotominor.yy118, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); -} - break; - case 200: /* expr ::= expr AND expr */ - case 201: /* expr ::= expr OR expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==202); - case 203: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==203); - case 204: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==204); - case 205: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==205); - case 206: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==206); - case 207: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==207); -{spanBinaryExpr(&yygotominor.yy118,pParse,yymsp[-1].major,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy118);} - break; - case 208: /* likeop ::= LIKE_KW */ - case 210: /* likeop ::= MATCH */ yytestcase(yyruleno==210); -{yygotominor.yy342.eOperator = yymsp[0].minor.yy0; yygotominor.yy342.not = 0;} - break; - case 209: /* likeop ::= NOT LIKE_KW */ - case 211: /* likeop ::= NOT MATCH */ yytestcase(yyruleno==211); -{yygotominor.yy342.eOperator = yymsp[0].minor.yy0; yygotominor.yy342.not = 1;} - break; - case 212: /* expr ::= expr likeop expr */ -{ - ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy118.pExpr); - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy342.eOperator); - if( yymsp[-1].minor.yy342.not ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-2].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; - if( yygotominor.yy118.pExpr ) yygotominor.yy118.pExpr->flags |= EP_InfixFunc; -} - break; - case 213: /* expr ::= expr likeop expr ESCAPE expr */ -{ - ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy118.pExpr); - yygotominor.yy118.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy342.eOperator); - if( yymsp[-3].minor.yy342.not ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; - if( yygotominor.yy118.pExpr ) yygotominor.yy118.pExpr->flags |= EP_InfixFunc; -} - break; - case 214: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(&yygotominor.yy118,pParse,yymsp[0].major,&yymsp[-1].minor.yy118,&yymsp[0].minor.yy0);} - break; - case 215: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(&yygotominor.yy118,pParse,TK_NOTNULL,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy0);} - break; - case 216: /* expr ::= expr IS expr */ -{ - spanBinaryExpr(&yygotominor.yy118,pParse,TK_IS,&yymsp[-2].minor.yy118,&yymsp[0].minor.yy118); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy118.pExpr, yygotominor.yy118.pExpr, TK_ISNULL); -} - break; - case 217: /* expr ::= expr IS NOT expr */ -{ - spanBinaryExpr(&yygotominor.yy118,pParse,TK_ISNOT,&yymsp[-3].minor.yy118,&yymsp[0].minor.yy118); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy118.pExpr, yygotominor.yy118.pExpr, TK_NOTNULL); + return TK_ID; } - break; - case 218: /* expr ::= NOT expr */ - case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219); -{spanUnaryPrefix(&yygotominor.yy118,pParse,yymsp[-1].major,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} - break; - case 220: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yygotominor.yy118,pParse,TK_UMINUS,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} - break; - case 221: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yygotominor.yy118,pParse,TK_UPLUS,&yymsp[0].minor.yy118,&yymsp[-1].minor.yy0);} - break; - case 224: /* expr ::= expr between_op expr AND expr */ -{ - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy118.pExpr); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pList = pList; - }else{ - sqlite3ExprListDelete(pParse->db, pList); - } - if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy118.zEnd; +SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ + return keywordCode((char*)z, n); } - break; - case 227: /* expr ::= expr in_op LP exprlist RP */ -{ - if( yymsp[-1].minor.yy322==0 ){ - /* Expressions of the form - ** - ** expr1 IN () - ** expr1 NOT IN () - ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. - */ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy4]); - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy118.pExpr); - }else{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pList = yymsp[-1].minor.yy322; - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); +#define SQLITE_N_KEYWORD 121 + +/************** End of keywordhash.h *****************************************/ +/************** Continuing where we left off in tokenize.c *******************/ + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identfiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ +}; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif + + +/* +** Return the length of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( *z ){ + case ' ': case '\t': case '\n': case '\f': case '\r': { + testcase( z[0]==' ' ); + testcase( z[0]=='\t' ); + testcase( z[0]=='\n' ); + testcase( z[0]=='\f' ); + testcase( z[0]=='\r' ); + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case '-': { + if( z[1]=='-' ){ + /* IMP: R-15891-05542 -- syntax diagram for comments */ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case '(': { + *tokenType = TK_LP; + return 1; + } + case ')': { + *tokenType = TK_RP; + return 1; + } + case ';': { + *tokenType = TK_SEMI; + return 1; + } + case '+': { + *tokenType = TK_PLUS; + return 1; + } + case '*': { + *tokenType = TK_STAR; + return 1; + } + case '/': { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + /* IMP: R-15891-05542 -- syntax diagram for comments */ + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + case '%': { + *tokenType = TK_REM; + return 1; + } + case '=': { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case '<': { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case '>': { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case '!': { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 2; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case '|': { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case ',': { + *tokenType = TK_COMMA; + return 1; + } + case '&': { + *tokenType = TK_BITAND; + return 1; + } + case '~': { + *tokenType = TK_BITNOT; + return 1; + } + case '`': + case '\'': + case '"': { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + *tokenType = TK_ILLEGAL; + return i; } - if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); } - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 228: /* expr ::= LP select RP */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pSelect = yymsp[-1].minor.yy387; - ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + case '.': { +#ifndef SQLITE_OMIT_FLOATING_POINT + if( !sqlite3Isdigit(z[1]) ) +#endif + { + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ } - yygotominor.yy118.zStart = yymsp[-2].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 229: /* expr ::= expr in_op LP select RP */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pSelect = yymsp[-1].minor.yy387; - ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); + testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); + testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); + testcase( z[0]=='9' ); + *tokenType = TK_INTEGER; + for(i=0; sqlite3Isdigit(z[i]); i++){} +#ifndef SQLITE_OMIT_FLOATING_POINT + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } +#endif + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; } - if( yymsp[-3].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-4].minor.yy118.zStart; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 230: /* expr ::= expr in_op nm dbnm */ -{ - SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy118.pExpr, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(yygotominor.yy118.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); - }else{ - sqlite3SrcListDelete(pParse->db, pSrc); + case '[': { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; } - if( yymsp[-2].minor.yy4 ) yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy118.pExpr, 0, 0); - yygotominor.yy118.zStart = yymsp[-3].minor.yy118.zStart; - yygotominor.yy118.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; - } - break; - case 231: /* expr ::= EXISTS LP select RP */ -{ - Expr *p = yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - if( p ){ - p->x.pSelect = yymsp[-1].minor.yy387; - ExprSetProperty(p, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, p); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy387); + case '?': { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case '#': { + for(i=1; sqlite3Isdigit(z[i]); i++){} + if( i>1 ){ + /* Parameters of the form #NNN (where NNN is a number) are used + ** internally by sqlite3NestedParse. */ + *tokenType = TK_REGISTER; + return i; + } + /* Fall through into the next case if the '#' is not followed by + ** a digit. Try to match #AAAA where AAAA is a parameter name. */ + } +#ifndef SQLITE_OMIT_TCL_VARIABLE + case '$': +#endif + case '@': /* For compatibility with MS SQL Server */ + case ':': { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; +#ifndef SQLITE_OMIT_TCL_VARIABLE + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; +#endif + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case 'x': case 'X': { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } + /* Otherwise fall through to the next case */ } - yygotominor.yy118.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 232: /* expr ::= CASE case_operand case_exprlist case_else END */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy314, yymsp[-1].minor.yy314, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->x.pList = yymsp[-2].minor.yy322; - sqlite3ExprSetHeight(pParse, yygotominor.yy118.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy322); - } - yygotominor.yy118.zStart = yymsp[-4].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 233: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ -{ - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy118.pExpr); - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yygotominor.yy322, yymsp[0].minor.yy118.pExpr); -} - break; - case 234: /* case_exprlist ::= WHEN expr THEN expr */ -{ - yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy118.pExpr); - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yygotominor.yy322, yymsp[0].minor.yy118.pExpr); -} - break; - case 243: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP */ -{ - sqlite3CreateIndex(pParse, &yymsp[-6].minor.yy0, &yymsp[-5].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-3].minor.yy0,0), yymsp[-1].minor.yy322, yymsp[-9].minor.yy4, - &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy4); -} - break; - case 244: /* uniqueflag ::= UNIQUE */ - case 298: /* raisetype ::= ABORT */ yytestcase(yyruleno==298); -{yygotominor.yy4 = OE_Abort;} - break; - case 245: /* uniqueflag ::= */ -{yygotominor.yy4 = OE_None;} - break; - case 248: /* idxlist ::= idxlist COMMA nm collate sortorder */ -{ - Expr *p = 0; - if( yymsp[-1].minor.yy0.n>0 ){ - p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); - } - yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, p); - sqlite3ExprListSetName(pParse,yygotominor.yy322,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy322, "index"); - if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; -} - break; - case 249: /* idxlist ::= nm collate sortorder */ -{ - Expr *p = 0; - if( yymsp[-1].minor.yy0.n>0 ){ - p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); - } - yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy322, "index"); - if( yygotominor.yy322 ) yygotominor.yy322->a[yygotominor.yy322->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy4; -} - break; - case 250: /* collate ::= */ -{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} - break; - case 252: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy259, yymsp[-1].minor.yy4);} - break; - case 253: /* cmd ::= VACUUM */ - case 254: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==254); -{sqlite3Vacuum(pParse);} - break; - case 255: /* cmd ::= PRAGMA nm dbnm */ -{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} - break; - case 256: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} - break; - case 257: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} - break; - case 258: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} - break; - case 259: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} - break; - case 270: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ -{ - Token all; - all.z = yymsp[-3].minor.yy0.z; - all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy203, &all); -} - break; - case 271: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ -{ - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy4, yymsp[-4].minor.yy90.a, yymsp[-4].minor.yy90.b, yymsp[-2].minor.yy259, yymsp[0].minor.yy314, yymsp[-10].minor.yy4, yymsp[-8].minor.yy4); - yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); -} - break; - case 272: /* trigger_time ::= BEFORE */ - case 275: /* trigger_time ::= */ yytestcase(yyruleno==275); -{ yygotominor.yy4 = TK_BEFORE; } - break; - case 273: /* trigger_time ::= AFTER */ -{ yygotominor.yy4 = TK_AFTER; } - break; - case 274: /* trigger_time ::= INSTEAD OF */ -{ yygotominor.yy4 = TK_INSTEAD;} - break; - case 276: /* trigger_event ::= DELETE|INSERT */ - case 277: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==277); -{yygotominor.yy90.a = yymsp[0].major; yygotominor.yy90.b = 0;} - break; - case 278: /* trigger_event ::= UPDATE OF inscollist */ -{yygotominor.yy90.a = TK_UPDATE; yygotominor.yy90.b = yymsp[0].minor.yy384;} - break; - case 281: /* when_clause ::= */ - case 303: /* key_opt ::= */ yytestcase(yyruleno==303); -{ yygotominor.yy314 = 0; } - break; - case 282: /* when_clause ::= WHEN expr */ - case 304: /* key_opt ::= KEY expr */ yytestcase(yyruleno==304); -{ yygotominor.yy314 = yymsp[0].minor.yy118.pExpr; } - break; - case 283: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ -{ - assert( yymsp[-2].minor.yy203!=0 ); - yymsp[-2].minor.yy203->pLast->pNext = yymsp[-1].minor.yy203; - yymsp[-2].minor.yy203->pLast = yymsp[-1].minor.yy203; - yygotominor.yy203 = yymsp[-2].minor.yy203; -} - break; - case 284: /* trigger_cmd_list ::= trigger_cmd SEMI */ -{ - assert( yymsp[-1].minor.yy203!=0 ); - yymsp[-1].minor.yy203->pLast = yymsp[-1].minor.yy203; - yygotominor.yy203 = yymsp[-1].minor.yy203; -} - break; - case 286: /* trnm ::= nm DOT nm */ -{ - yygotominor.yy0 = yymsp[0].minor.yy0; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); -} - break; - case 288: /* tridxby ::= INDEXED BY nm */ -{ - sqlite3ErrorMsg(pParse, - "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - break; - case 289: /* tridxby ::= NOT INDEXED */ -{ - sqlite3ErrorMsg(pParse, - "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - break; - case 290: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{ yygotominor.yy203 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy322, yymsp[0].minor.yy314, yymsp[-5].minor.yy210); } - break; - case 291: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP */ -{yygotominor.yy203 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy384, yymsp[-1].minor.yy322, 0, yymsp[-7].minor.yy210);} - break; - case 292: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ -{yygotominor.yy203 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy384, 0, yymsp[0].minor.yy387, yymsp[-4].minor.yy210);} - break; - case 293: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yygotominor.yy203 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy314);} - break; - case 294: /* trigger_cmd ::= select */ -{yygotominor.yy203 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy387); } - break; - case 295: /* expr ::= RAISE LP IGNORE RP */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yygotominor.yy118.pExpr ){ - yygotominor.yy118.pExpr->affinity = OE_Ignore; - } - yygotominor.yy118.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 296: /* expr ::= RAISE LP raisetype COMMA nm RP */ -{ - yygotominor.yy118.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yygotominor.yy118.pExpr ) { - yygotominor.yy118.pExpr->affinity = (char)yymsp[-3].minor.yy4; - } - yygotominor.yy118.zStart = yymsp[-5].minor.yy0.z; - yygotominor.yy118.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 297: /* raisetype ::= ROLLBACK */ -{yygotominor.yy4 = OE_Rollback;} - break; - case 299: /* raisetype ::= FAIL */ -{yygotominor.yy4 = OE_Fail;} - break; - case 300: /* cmd ::= DROP TRIGGER ifexists fullname */ -{ - sqlite3DropTrigger(pParse,yymsp[0].minor.yy259,yymsp[-1].minor.yy4); -} - break; - case 301: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ -{ - sqlite3Attach(pParse, yymsp[-3].minor.yy118.pExpr, yymsp[-1].minor.yy118.pExpr, yymsp[0].minor.yy314); -} - break; - case 302: /* cmd ::= DETACH database_kw_opt expr */ -{ - sqlite3Detach(pParse, yymsp[0].minor.yy118.pExpr); -} - break; - case 307: /* cmd ::= REINDEX */ -{sqlite3Reindex(pParse, 0, 0);} - break; - case 308: /* cmd ::= REINDEX nm dbnm */ -{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 309: /* cmd ::= ANALYZE */ -{sqlite3Analyze(pParse, 0, 0);} - break; - case 310: /* cmd ::= ANALYZE nm dbnm */ -{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 311: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ -{ - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy259,&yymsp[0].minor.yy0); -} - break; - case 312: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ -{ - sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); -} - break; - case 313: /* add_column_fullname ::= fullname */ -{ - pParse->db->lookaside.bEnabled = 0; - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy259); -} - break; - case 316: /* cmd ::= create_vtab */ -{sqlite3VtabFinishParse(pParse,0);} - break; - case 317: /* cmd ::= create_vtab LP vtabarglist RP */ -{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} - break; - case 318: /* create_vtab ::= createkw VIRTUAL TABLE nm dbnm USING nm */ -{ - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); -} - break; - case 321: /* vtabarg ::= */ -{sqlite3VtabArgInit(pParse);} - break; - case 323: /* vtabargtoken ::= ANY */ - case 324: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==324); - case 325: /* lp ::= LP */ yytestcase(yyruleno==325); -{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} - break; - default: - /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); - /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); - /* (2) cmdlist ::= ecmd */ yytestcase(yyruleno==2); - /* (3) ecmd ::= SEMI */ yytestcase(yyruleno==3); - /* (4) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==4); - /* (10) trans_opt ::= */ yytestcase(yyruleno==10); - /* (11) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==11); - /* (12) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==12); - /* (20) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==20); - /* (21) savepoint_opt ::= */ yytestcase(yyruleno==21); - /* (25) cmd ::= create_table create_table_args */ yytestcase(yyruleno==25); - /* (34) columnlist ::= columnlist COMMA column */ yytestcase(yyruleno==34); - /* (35) columnlist ::= column */ yytestcase(yyruleno==35); - /* (44) type ::= */ yytestcase(yyruleno==44); - /* (51) signed ::= plus_num */ yytestcase(yyruleno==51); - /* (52) signed ::= minus_num */ yytestcase(yyruleno==52); - /* (53) carglist ::= carglist carg */ yytestcase(yyruleno==53); - /* (54) carglist ::= */ yytestcase(yyruleno==54); - /* (55) carg ::= CONSTRAINT nm ccons */ yytestcase(yyruleno==55); - /* (56) carg ::= ccons */ yytestcase(yyruleno==56); - /* (62) ccons ::= NULL onconf */ yytestcase(yyruleno==62); - /* (90) conslist ::= conslist COMMA tcons */ yytestcase(yyruleno==90); - /* (91) conslist ::= conslist tcons */ yytestcase(yyruleno==91); - /* (92) conslist ::= tcons */ yytestcase(yyruleno==92); - /* (93) tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==93); - /* (268) plus_opt ::= PLUS */ yytestcase(yyruleno==268); - /* (269) plus_opt ::= */ yytestcase(yyruleno==269); - /* (279) foreach_clause ::= */ yytestcase(yyruleno==279); - /* (280) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==280); - /* (287) tridxby ::= */ yytestcase(yyruleno==287); - /* (305) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==305); - /* (306) database_kw_opt ::= */ yytestcase(yyruleno==306); - /* (314) kwcolumn_opt ::= */ yytestcase(yyruleno==314); - /* (315) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==315); - /* (319) vtabarglist ::= vtabarg */ yytestcase(yyruleno==319); - /* (320) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==320); - /* (322) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==322); - /* (326) anylist ::= */ yytestcase(yyruleno==326); - /* (327) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==327); - /* (328) anylist ::= anylist ANY */ yytestcase(yyruleno==328); - break; - }; - yygoto = yyRuleInfo[yyruleno].lhs; - yysize = yyRuleInfo[yyruleno].nrhs; - yypParser->yyidx -= yysize; - yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least - ** one element off the stack, then we can push the new element back - ** onto the stack here, and skip the stack overflow test in yy_shift(). - ** That gives a significant speed improvement. */ - if( yysize ){ - yypParser->yyidx++; - yymsp -= yysize-1; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yymsp->minor = yygotominor; - }else #endif - { - yy_shift(yypParser,yyact,yygoto,&yygotominor); + default: { + if( !IdChar(*z) ){ + break; + } + for(i=1; IdChar(z[i]); i++){} + *tokenType = keywordCode((char*)z, i); + return i; } - }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); - yy_accept(yypParser); } + *tokenType = TK_ILLEGAL; + return 1; } /* -** The following code executes when the parse fails +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. */ -#ifndef YYNOERRORRECOVERY -static void yy_parse_failed( - yyParser *yypParser /* The parser */ -){ - sqlite3ParserARG_FETCH; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + int i; /* Loop counter */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ + + + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->activeVdbeCnt==0 ){ + db->u1.isInterrupted = 0; + } + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + i = 0; + assert( pzErrMsg!=0 ); + pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); + if( pEngine==0 ){ + db->mallocFailed = 1; + return SQLITE_NOMEM; + } + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); + enableLookaside = db->lookaside.bEnabled; + if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; + while( !db->mallocFailed && zSql[i]!=0 ){ + assert( i>=0 ); + pParse->sLastToken.z = &zSql[i]; + pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>mxSqlLen ){ + pParse->rc = SQLITE_TOOBIG; + break; + } + switch( tokenType ){ + case TK_SPACE: { + if( db->u1.isInterrupted ){ + sqlite3ErrorMsg(pParse, "interrupt"); + pParse->rc = SQLITE_INTERRUPT; + goto abort_parse; + } + break; + } + case TK_ILLEGAL: { + sqlite3DbFree(db, *pzErrMsg); + *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", + &pParse->sLastToken); + nErr++; + goto abort_parse; + } + case TK_SEMI: { + pParse->zTail = &zSql[i]; + /* Fall thru into the default case */ + } + default: { + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK ){ + goto abort_parse; + } + break; + } + } + } +abort_parse: + if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ + if( lastTokenParsed!=TK_SEMI ){ + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; + } + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + } +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); +#endif /* YYDEBUG */ + sqlite3ParserFree(pEngine, sqlite3_free); + db->lookaside.bEnabled = enableLookaside; + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); + } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s", *pzErrMsg); + pParse->zErrMsg = 0; + nErr++; + } + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser fails */ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif + + if( !IN_DECLARE_VTAB ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(db, pParse->pNewTable); + } + + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); + sqlite3DbFree(db, pParse->aAlias); + while( pParse->pAinc ){ + AutoincInfo *p = pParse->pAinc; + pParse->pAinc = p->pNext; + sqlite3DbFree(db, p); + } + while( pParse->pZombieTab ){ + Table *p = pParse->pZombieTab; + pParse->pZombieTab = p->pNextZombie; + sqlite3DeleteTable(db, p); + } + if( nErr>0 && pParse->rc==SQLITE_OK ){ + pParse->rc = SQLITE_ERROR; + } + return nErr; } -#endif /* YYNOERRORRECOVERY */ +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ /* -** The following code executes when a syntax error first occurs. +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. */ -static void yy_syntax_error( - yyParser *yypParser, /* The parser */ - int yymajor, /* The major type of the error token */ - YYMINORTYPE yyminor /* The minor type of the error token */ -){ - sqlite3ParserARG_FETCH; -#define TOKEN (yyminor.yy0) +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ - UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ - assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); - pParse->parseError = 1; - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} /* -** The following is executed when the parser accepts +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. */ -static void yy_accept( - yyParser *yypParser /* The parser */ -){ - sqlite3ParserARG_FETCH; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); - } +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser accepts */ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} -/* The main parser program. -** The first argument is a pointer to a structure obtained from -** "sqlite3ParserAlloc" which describes the current state of the parser. -** The second argument is the major token number. The third is -** the minor token. The fourth optional argument is whatever the -** user wants (and specified in the grammar) and is available for -** use by the action routines. +/* +** Return TRUE if the given SQL string ends in a semicolon. ** -** Inputs: -**
      -**
    • A pointer to the parser (an opaque structure.) -**
    • The major token number. -**
    • The minor token number. -**
    • An option argument of a grammar-specified type. -**
    +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". ** -** Outputs: -** None. +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceeded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger difinition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. */ -SQLITE_PRIVATE void sqlite3Parser( - void *yyp, /* The parser */ - int yymajor, /* The major token code number */ - sqlite3ParserTOKENTYPE yyminor /* The value for the token */ - sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ -){ - YYMINORTYPE yyminorunion; - int yyact; /* The parser action. */ - int yyendofinput; /* True if we are at the end of input */ -#ifdef YYERRORSYMBOL - int yyerrorhit = 0; /* True if yymajor has invoked an error */ -#endif - yyParser *yypParser; /* The parser */ - - /* (re)initialize the parser, if necessary */ - yypParser = (yyParser*)yyp; - if( yypParser->yyidx<0 ){ -#if YYSTACKDEPTH<=0 - if( yypParser->yystksz <=0 ){ - /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ - yyminorunion = yyzerominor; - yyStackOverflow(yypParser, &yyminorunion); - return; - } -#endif - yypParser->yyidx = 0; - yypParser->yyerrcnt = -1; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; - } - yyminorunion.yy0 = yyminor; - yyendofinput = (yymajor==0); - sqlite3ParserARG_STORE; +SQLITE_API int sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); - } -#endif +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simplier + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ - do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyactyyerrcnt--; - yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); - }else{ - assert( yyact == YY_ERROR_ACTION ); -#ifdef YYERRORSYMBOL - int yymx; -#endif -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; } -#endif -#ifdef YYERRORSYMBOL - /* A syntax error has occurred. - ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". - ** - ** This is what we do if the grammar does define ERROR: - ** - ** * Call the %syntax_error function. - ** - ** * Begin popping the stack until we enter a state where - ** it is legal to shift the error symbol, then shift - ** the error symbol. - ** - ** * Set the error count to three. - ** - ** * Begin accepting and shifting new tokens. No new error - ** processing will occur until three tokens have been - ** shifted successfully. - ** - */ - if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; } - yymx = yypParser->yystack[yypParser->yyidx].major; - if( yymx==YYERRORSYMBOL || yyerrorhit ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sDiscard input token %s\n", - yyTracePrompt,yyTokenName[yymajor]); - } -#endif - yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - }else{ - while( - yypParser->yyidx >= 0 && - yymx != YYERRORSYMBOL && - (yyact = yy_find_reduce_action( - yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE - ){ - yy_pop_parser_stack(yypParser); + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; } - if( yypParser->yyidx < 0 || yymajor==0 ){ - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yy_parse_failed(yypParser); - yymajor = YYNOCODE; - }else if( yymx!=YYERRORSYMBOL ){ - YYMINORTYPE u2; - u2.YYERRSYMDT = 0; - yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; } - yypParser->yyerrcnt = 3; - yyerrorhit = 1; -#elif defined(YYNOERRORRECOVERY) - /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to - ** do any kind of error recovery. Instead, simply invoke the syntax - ** error routine and continue going as if nothing had happened. - ** - ** Applications can set this macro (for example inside %include) if - ** they intend to abandon the parse upon the first syntax error seen. - */ - yy_syntax_error(yypParser,yymajor,yyminorunion); - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - -#else /* YYERRORSYMBOL is not defined */ - /* This is what we do if the grammar does not define ERROR: - ** - ** * Report an error message, and throw away the input token. - ** - ** * If the input token is $, then fail the parse. - ** - ** As before, subsequent error messages are suppressed until - ** three input tokens have been successfully shifted. - */ - if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; } - yypParser->yyerrcnt = 3; - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - if( yyendofinput ){ - yy_parse_failed(yypParser); + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; } - yymajor = YYNOCODE; + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else #endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } } - }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); - return; + state = trans[state][token]; + zSql++; + } + return state==1; } -/************** End of parse.c ***********************************************/ -/************** Begin file tokenize.c ****************************************/ +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +SQLITE_API int sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc = SQLITE_NOMEM; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + return sqlite3ApiExit(0, rc); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ + +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ /* ** 2001 September 15 ** @@ -103489,8238 +107844,9073 @@ SQLITE_PRIVATE void sqlite3Parser( ** May you share freely, never taking more than you give. ** ************************************************************************* -** An tokenizer for SQL -** -** This file contains C code that splits an SQL input string up into -** individual tokens and sends those tokens one-by-one over to the -** parser for analysis. +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. */ +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ /* -** The charMap() macro maps alphabetic characters into their -** lower-case ASCII equivalent. On ASCII machines, this is just -** an upper-to-lower case map. On EBCDIC machines we also need -** to adjust the encoding. Only alphabetic characters and underscores -** need to be translated. +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. */ -#ifdef SQLITE_ASCII -# define charMap(X) sqlite3UpperToLower[(unsigned char)X] -#endif -#ifdef SQLITE_EBCDIC -# define charMap(X) ebcdicToAscii[(unsigned char)X] -const unsigned char ebcdicToAscii[] = { -/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ - 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ - 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ - 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ - 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ - 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ - 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ -}; -#endif +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ /* -** The sqlite3KeywordCode function looks up an identifier to determine if -** it is a keyword. If it is a keyword, the token code of that keyword is -** returned. If the input is not a keyword, TK_ID is returned. +** 2008 May 26 ** -** The implementation of this routine was generated by a program, -** mkkeywordhash.h, located in the tool subdirectory of the distribution. -** The output of the mkkeywordhash.c program is written into a file -** named keywordhash.h and then included into this source file by -** the #include below. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. */ -/************** Include keywordhash.h in the middle of tokenize.c ************/ -/************** Begin file keywordhash.h *************************************/ -/***** This file contains automatically generated code ****** + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_ICU +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26 ** -** The code in this file has been automatically generated by +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** sqlite/tool/mkkeywordhash.c +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** The code in this file implements a function that determines whether -** or not a given identifier is really an SQL keyword. The same thing -** might be implemented more directly using a hand-written hash table. -** But by using this automatically generated code, the size of the code -** is substantially reduced. This is important for embedded applications -** on platforms with limited memory. +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. */ -/* Hash score: 175 */ -static int keywordCode(const char *z, int n){ - /* zText[] encodes 811 bytes of keywords in 541 bytes */ - /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ - /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ - /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ - /* UNIQUERYATTACHAVINGROUPDATEBEGINNERELEASEBETWEENOTNULLIKE */ - /* CASCADELETECASECOLLATECREATECURRENT_DATEDETACHIMMEDIATEJOIN */ - /* SERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHENWHERENAME */ - /* AFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSS */ - /* CURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAILFROMFULLGLOBYIF */ - /* ISNULLORDERESTRICTOUTERIGHTROLLBACKROWUNIONUSINGVACUUMVIEW */ - /* INITIALLY */ - static const char zText[540] = { - 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', - 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', - 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', - 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', - 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', - 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', - 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', - 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', - 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', - 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', - 'U','E','R','Y','A','T','T','A','C','H','A','V','I','N','G','R','O','U', - 'P','D','A','T','E','B','E','G','I','N','N','E','R','E','L','E','A','S', - 'E','B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C', - 'A','S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L', - 'A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D', - 'A','T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E', - 'J','O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A', - 'L','Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U', - 'E','S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W', - 'H','E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C', - 'E','A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R', - 'E','M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M', - 'M','I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U', - 'R','R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M', - 'A','R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T', - 'D','R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L', - 'O','B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S', - 'T','R','I','C','T','O','U','T','E','R','I','G','H','T','R','O','L','L', - 'B','A','C','K','R','O','W','U','N','I','O','N','U','S','I','N','G','V', - 'A','C','U','U','M','V','I','E','W','I','N','I','T','I','A','L','L','Y', - }; - static const unsigned char aHash[127] = { - 72, 101, 114, 70, 0, 45, 0, 0, 78, 0, 73, 0, 0, - 42, 12, 74, 15, 0, 113, 81, 50, 108, 0, 19, 0, 0, - 118, 0, 116, 111, 0, 22, 89, 0, 9, 0, 0, 66, 67, - 0, 65, 6, 0, 48, 86, 98, 0, 115, 97, 0, 0, 44, - 0, 99, 24, 0, 17, 0, 119, 49, 23, 0, 5, 106, 25, - 92, 0, 0, 121, 102, 56, 120, 53, 28, 51, 0, 87, 0, - 96, 26, 0, 95, 0, 0, 0, 91, 88, 93, 84, 105, 14, - 39, 104, 0, 77, 0, 18, 85, 107, 32, 0, 117, 76, 109, - 58, 46, 80, 0, 0, 90, 40, 0, 112, 0, 36, 0, 0, - 29, 0, 82, 59, 60, 0, 20, 57, 0, 52, - }; - static const unsigned char aNext[121] = { - 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 43, 3, 47, - 0, 0, 0, 0, 30, 0, 54, 0, 38, 0, 0, 0, 1, - 62, 0, 0, 63, 0, 41, 0, 0, 0, 0, 0, 0, 0, - 61, 0, 0, 0, 0, 31, 55, 16, 34, 10, 0, 0, 0, - 0, 0, 0, 0, 11, 68, 75, 0, 8, 0, 100, 94, 0, - 103, 0, 83, 0, 71, 0, 0, 110, 27, 37, 69, 79, 0, - 35, 64, 0, 0, - }; - static const unsigned char aLen[121] = { - 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, - 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, - 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, - 4, 6, 2, 3, 9, 4, 2, 6, 5, 6, 6, 5, 6, - 5, 5, 7, 7, 7, 3, 2, 4, 4, 7, 3, 6, 4, - 7, 6, 12, 6, 9, 4, 6, 5, 4, 7, 6, 5, 6, - 7, 5, 4, 5, 6, 5, 7, 3, 7, 13, 2, 2, 4, - 6, 6, 8, 5, 17, 12, 7, 8, 8, 2, 4, 4, 4, - 4, 4, 2, 2, 6, 5, 8, 5, 5, 8, 3, 5, 5, - 6, 4, 9, 3, - }; - static const unsigned short int aOffset[121] = { - 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, - 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, - 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, - 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 189, 194, 197, - 203, 206, 210, 217, 223, 223, 223, 226, 229, 233, 234, 238, 244, - 248, 255, 261, 273, 279, 288, 290, 296, 301, 303, 310, 315, 320, - 326, 332, 337, 341, 344, 350, 354, 361, 363, 370, 372, 374, 383, - 387, 393, 399, 407, 412, 412, 428, 435, 442, 443, 450, 454, 458, - 462, 466, 469, 471, 473, 479, 483, 491, 495, 500, 508, 511, 516, - 521, 527, 531, 536, - }; - static const unsigned char aCode[121] = { - TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, - TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, - TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, - TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, - TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, - TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, - TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, - TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, - TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, - TK_OR, TK_UNIQUE, TK_QUERY, TK_ATTACH, TK_HAVING, - TK_GROUP, TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RELEASE, - TK_BETWEEN, TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, - TK_LIKE_KW, TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, - TK_COLLATE, TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, - TK_JOIN, TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, - TK_PRAGMA, TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, - TK_WHEN, TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, - TK_AND, TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, - TK_CAST, TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, - TK_CTIME_KW, TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, - TK_IS, TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, - TK_LIKE_KW, TK_BY, TK_IF, TK_ISNULL, TK_ORDER, - TK_RESTRICT, TK_JOIN_KW, TK_JOIN_KW, TK_ROLLBACK, TK_ROW, - TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, - TK_ALL, - }; - int h, i; - if( n<2 ) return TK_ID; - h = ((charMap(z[0])*4) ^ - (charMap(z[n-1])*3) ^ - n) % 127; - for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ - if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ - testcase( i==0 ); /* REINDEX */ - testcase( i==1 ); /* INDEXED */ - testcase( i==2 ); /* INDEX */ - testcase( i==3 ); /* DESC */ - testcase( i==4 ); /* ESCAPE */ - testcase( i==5 ); /* EACH */ - testcase( i==6 ); /* CHECK */ - testcase( i==7 ); /* KEY */ - testcase( i==8 ); /* BEFORE */ - testcase( i==9 ); /* FOREIGN */ - testcase( i==10 ); /* FOR */ - testcase( i==11 ); /* IGNORE */ - testcase( i==12 ); /* REGEXP */ - testcase( i==13 ); /* EXPLAIN */ - testcase( i==14 ); /* INSTEAD */ - testcase( i==15 ); /* ADD */ - testcase( i==16 ); /* DATABASE */ - testcase( i==17 ); /* AS */ - testcase( i==18 ); /* SELECT */ - testcase( i==19 ); /* TABLE */ - testcase( i==20 ); /* LEFT */ - testcase( i==21 ); /* THEN */ - testcase( i==22 ); /* END */ - testcase( i==23 ); /* DEFERRABLE */ - testcase( i==24 ); /* ELSE */ - testcase( i==25 ); /* EXCEPT */ - testcase( i==26 ); /* TRANSACTION */ - testcase( i==27 ); /* ACTION */ - testcase( i==28 ); /* ON */ - testcase( i==29 ); /* NATURAL */ - testcase( i==30 ); /* ALTER */ - testcase( i==31 ); /* RAISE */ - testcase( i==32 ); /* EXCLUSIVE */ - testcase( i==33 ); /* EXISTS */ - testcase( i==34 ); /* SAVEPOINT */ - testcase( i==35 ); /* INTERSECT */ - testcase( i==36 ); /* TRIGGER */ - testcase( i==37 ); /* REFERENCES */ - testcase( i==38 ); /* CONSTRAINT */ - testcase( i==39 ); /* INTO */ - testcase( i==40 ); /* OFFSET */ - testcase( i==41 ); /* OF */ - testcase( i==42 ); /* SET */ - testcase( i==43 ); /* TEMPORARY */ - testcase( i==44 ); /* TEMP */ - testcase( i==45 ); /* OR */ - testcase( i==46 ); /* UNIQUE */ - testcase( i==47 ); /* QUERY */ - testcase( i==48 ); /* ATTACH */ - testcase( i==49 ); /* HAVING */ - testcase( i==50 ); /* GROUP */ - testcase( i==51 ); /* UPDATE */ - testcase( i==52 ); /* BEGIN */ - testcase( i==53 ); /* INNER */ - testcase( i==54 ); /* RELEASE */ - testcase( i==55 ); /* BETWEEN */ - testcase( i==56 ); /* NOTNULL */ - testcase( i==57 ); /* NOT */ - testcase( i==58 ); /* NO */ - testcase( i==59 ); /* NULL */ - testcase( i==60 ); /* LIKE */ - testcase( i==61 ); /* CASCADE */ - testcase( i==62 ); /* ASC */ - testcase( i==63 ); /* DELETE */ - testcase( i==64 ); /* CASE */ - testcase( i==65 ); /* COLLATE */ - testcase( i==66 ); /* CREATE */ - testcase( i==67 ); /* CURRENT_DATE */ - testcase( i==68 ); /* DETACH */ - testcase( i==69 ); /* IMMEDIATE */ - testcase( i==70 ); /* JOIN */ - testcase( i==71 ); /* INSERT */ - testcase( i==72 ); /* MATCH */ - testcase( i==73 ); /* PLAN */ - testcase( i==74 ); /* ANALYZE */ - testcase( i==75 ); /* PRAGMA */ - testcase( i==76 ); /* ABORT */ - testcase( i==77 ); /* VALUES */ - testcase( i==78 ); /* VIRTUAL */ - testcase( i==79 ); /* LIMIT */ - testcase( i==80 ); /* WHEN */ - testcase( i==81 ); /* WHERE */ - testcase( i==82 ); /* RENAME */ - testcase( i==83 ); /* AFTER */ - testcase( i==84 ); /* REPLACE */ - testcase( i==85 ); /* AND */ - testcase( i==86 ); /* DEFAULT */ - testcase( i==87 ); /* AUTOINCREMENT */ - testcase( i==88 ); /* TO */ - testcase( i==89 ); /* IN */ - testcase( i==90 ); /* CAST */ - testcase( i==91 ); /* COLUMN */ - testcase( i==92 ); /* COMMIT */ - testcase( i==93 ); /* CONFLICT */ - testcase( i==94 ); /* CROSS */ - testcase( i==95 ); /* CURRENT_TIMESTAMP */ - testcase( i==96 ); /* CURRENT_TIME */ - testcase( i==97 ); /* PRIMARY */ - testcase( i==98 ); /* DEFERRED */ - testcase( i==99 ); /* DISTINCT */ - testcase( i==100 ); /* IS */ - testcase( i==101 ); /* DROP */ - testcase( i==102 ); /* FAIL */ - testcase( i==103 ); /* FROM */ - testcase( i==104 ); /* FULL */ - testcase( i==105 ); /* GLOB */ - testcase( i==106 ); /* BY */ - testcase( i==107 ); /* IF */ - testcase( i==108 ); /* ISNULL */ - testcase( i==109 ); /* ORDER */ - testcase( i==110 ); /* RESTRICT */ - testcase( i==111 ); /* OUTER */ - testcase( i==112 ); /* RIGHT */ - testcase( i==113 ); /* ROLLBACK */ - testcase( i==114 ); /* ROW */ - testcase( i==115 ); /* UNION */ - testcase( i==116 ); /* USING */ - testcase( i==117 ); /* VACUUM */ - testcase( i==118 ); /* VIEW */ - testcase( i==119 ); /* INITIALLY */ - testcase( i==120 ); /* ALL */ - return aCode[i]; - } - } - return TK_ID; -} -SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ - return keywordCode((char*)z, n); -} -#define SQLITE_N_KEYWORD 121 -/************** End of keywordhash.h *****************************************/ -/************** Continuing where we left off in tokenize.c *******************/ +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ +SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*, ...) = 0; +#endif /* -** If X is a character that can be used in an identifier then -** IdChar(X) will be true. Otherwise it is false. +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. ** -** For ASCII, any character with the high-order bit set is -** allowed in an identifier. For 7-bit characters, -** sqlite3IsIdChar[X] must be 1. +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_temp_directory = 0; + +/* +** Initialize SQLite. ** -** For EBCDIC, the rules are more complex but have the same -** end result. +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). ** -** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identfiers. But many SQL implementations do. -** SQLite will allow '$' in identifiers for compatibility. -** But the feature is undocumented. +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. */ -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) -#endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ - 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ -}; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +SQLITE_API int sqlite3_initialize(void){ + sqlite3_mutex *pMaster; /* The main static mutex */ + int rc; /* Result code */ + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } #endif + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; -/* -** Return the length of the token that begins at z[0]. -** Store the token type in *tokenType before returning. -*/ -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; - switch( *z ){ - case ' ': case '\t': case '\n': case '\f': case '\r': { - testcase( z[0]==' ' ); - testcase( z[0]=='\t' ); - testcase( z[0]=='\n' ); - testcase( z[0]=='\f' ); - testcase( z[0]=='\r' ); - for(i=1; sqlite3Isspace(z[i]); i++){} - *tokenType = TK_SPACE; - return i; - } - case '-': { - if( z[1]=='-' ){ - /* IMP: R-15891-05542 -- syntax diagram for comments */ - for(i=2; (c=z[i])!=0 && c!='\n'; i++){} - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ - return i; - } - *tokenType = TK_MINUS; - return 1; - } - case '(': { - *tokenType = TK_LP; - return 1; - } - case ')': { - *tokenType = TK_RP; - return 1; - } - case ';': { - *tokenType = TK_SEMI; - return 1; - } - case '+': { - *tokenType = TK_PLUS; - return 1; - } - case '*': { - *tokenType = TK_STAR; - return 1; - } - case '/': { - if( z[1]!='*' || z[2]==0 ){ - *tokenType = TK_SLASH; - return 1; + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM; } - /* IMP: R-15891-05542 -- syntax diagram for comments */ - for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} - if( c ) i++; - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ - return i; } - case '%': { - *tokenType = TK_REM; - return 1; + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMaster); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + sqlite3GlobalConfig.inProgress = 1; + memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); + sqlite3RegisterGlobalFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); } - case '=': { - *tokenType = TK_EQ; - return 1 + (z[1]=='='); + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); } - case '<': { - if( (c=z[1])=='=' ){ - *tokenType = TK_LE; - return 2; - }else if( c=='>' ){ - *tokenType = TK_NE; - return 2; - }else if( c=='<' ){ - *tokenType = TK_LSHIFT; - return 2; - }else{ - *tokenType = TK_LT; - return 1; - } + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3GlobalConfig.isInit = 1; } - case '>': { - if( (c=z[1])=='=' ){ - *tokenType = TK_GE; - return 2; - }else if( c=='>' ){ - *tokenType = TK_RSHIFT; - return 2; - }else{ - *tokenType = TK_GT; - return 1; - } + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +SQLITE_API int sqlite3_shutdown(void){ + if( sqlite3GlobalConfig.isInit ){ + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +SQLITE_API int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 + case SQLITE_CONFIG_SINGLETHREAD: { + /* Disable all mutexing */ + sqlite3GlobalConfig.bCoreMutex = 0; + sqlite3GlobalConfig.bFullMutex = 0; + break; } - case '!': { - if( z[1]!='=' ){ - *tokenType = TK_ILLEGAL; - return 2; - }else{ - *tokenType = TK_NE; - return 2; - } + case SQLITE_CONFIG_MULTITHREAD: { + /* Disable mutexing of database connections */ + /* Enable mutexing of core data structures */ + sqlite3GlobalConfig.bCoreMutex = 1; + sqlite3GlobalConfig.bFullMutex = 0; + break; } - case '|': { - if( z[1]!='|' ){ - *tokenType = TK_BITOR; - return 1; - }else{ - *tokenType = TK_CONCAT; - return 2; - } + case SQLITE_CONFIG_SERIALIZED: { + /* Enable all mutexing */ + sqlite3GlobalConfig.bCoreMutex = 1; + sqlite3GlobalConfig.bFullMutex = 1; + break; } - case ',': { - *tokenType = TK_COMMA; - return 1; + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; } - case '&': { - *tokenType = TK_BITAND; - return 1; + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; } - case '~': { - *tokenType = TK_BITNOT; - return 1; +#endif + + + case SQLITE_CONFIG_MALLOC: { + /* Specify an alternative malloc implementation */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; } - case '`': - case '\'': - case '"': { - int delim = z[0]; - testcase( delim=='`' ); - testcase( delim=='\'' ); - testcase( delim=='"' ); - for(i=1; (c=z[i])!=0; i++){ - if( c==delim ){ - if( z[i+1]==delim ){ - i++; - }else{ - break; - } - } - } - if( c=='\'' ){ - *tokenType = TK_STRING; - return i+1; - }else if( c!=0 ){ - *tokenType = TK_ID; - return i+1; - }else{ - *tokenType = TK_ILLEGAL; - return i; - } + case SQLITE_CONFIG_GETMALLOC: { + /* Retrieve the current malloc() implementation */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; } - case '.': { -#ifndef SQLITE_OMIT_FLOATING_POINT - if( !sqlite3Isdigit(z[1]) ) -#endif - { - *tokenType = TK_DOT; - return 1; - } - /* If the next character is a digit, this is a floating point - ** number that begins with ".". Fall thru into the next case */ + case SQLITE_CONFIG_MEMSTATUS: { + /* Enable or disable the malloc status collection */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; } - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': { - testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); - testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); - testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); - testcase( z[0]=='9' ); - *tokenType = TK_INTEGER; - for(i=0; sqlite3Isdigit(z[i]); i++){} -#ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } -#endif - while( IdChar(z[i]) ){ - *tokenType = TK_ILLEGAL; - i++; - } - return i; + case SQLITE_CONFIG_SCRATCH: { + /* Designate a buffer for scratch memory space */ + sqlite3GlobalConfig.pScratch = va_arg(ap, void*); + sqlite3GlobalConfig.szScratch = va_arg(ap, int); + sqlite3GlobalConfig.nScratch = va_arg(ap, int); + break; } - case '[': { - for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} - *tokenType = c==']' ? TK_ID : TK_ILLEGAL; - return i; + case SQLITE_CONFIG_PAGECACHE: { + /* Designate a buffer for page cache memory space */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; } - case '?': { - *tokenType = TK_VARIABLE; - for(i=1; sqlite3Isdigit(z[i]); i++){} - return i; + + case SQLITE_CONFIG_PCACHE: { + /* Specify an alternative page cache implementation */ + sqlite3GlobalConfig.pcache = *va_arg(ap, sqlite3_pcache_methods*); + break; } - case '#': { - for(i=1; sqlite3Isdigit(z[i]); i++){} - if( i>1 ){ - /* Parameters of the form #NNN (where NNN is a number) are used - ** internally by sqlite3NestedParse. */ - *tokenType = TK_REGISTER; - return i; + + case SQLITE_CONFIG_GETPCACHE: { + if( sqlite3GlobalConfig.pcache.xInit==0 ){ + sqlite3PCacheSetDefault(); } - /* Fall through into the next case if the '#' is not followed by - ** a digit. Try to match #AAAA where AAAA is a parameter name. */ + *va_arg(ap, sqlite3_pcache_methods*) = sqlite3GlobalConfig.pcache; + break; } -#ifndef SQLITE_OMIT_TCL_VARIABLE - case '$': + +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* Designate a buffer for heap memory space */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* If the heap pointer is NULL, then restore the malloc implementation + ** back to NULL pointers too. This will cause the malloc to go + ** back to its default implementation when sqlite3_initialize() is + ** run. + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* The heap pointer is not NULL, then install one of the + ** mem5.c/mem3.c methods. If neither ENABLE_MEMSYS3 nor + ** ENABLE_MEMSYS5 is defined, return an error. + */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); #endif - case '@': /* For compatibility with MS SQL Server */ - case ':': { - int n = 0; - testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); - *tokenType = TK_VARIABLE; - for(i=1; (c=z[i])!=0; i++){ - if( IdChar(c) ){ - n++; -#ifndef SQLITE_OMIT_TCL_VARIABLE - }else if( c=='(' && n>0 ){ - do{ - i++; - }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); - if( c==')' ){ - i++; - }else{ - *tokenType = TK_ILLEGAL; - } - break; - }else if( c==':' && z[i+1]==':' ){ - i++; +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); #endif - }else{ - break; - } - } - if( n==0 ) *tokenType = TK_ILLEGAL; - return i; - } -#ifndef SQLITE_OMIT_BLOB_LITERAL - case 'x': case 'X': { - testcase( z[0]=='x' ); testcase( z[0]=='X' ); - if( z[1]=='\'' ){ - *tokenType = TK_BLOB; - for(i=2; (c=z[i])!=0 && c!='\''; i++){ - if( !sqlite3Isxdigit(c) ){ - *tokenType = TK_ILLEGAL; - } - } - if( i%2 || !c ) *tokenType = TK_ILLEGAL; - if( c ) i++; - return i; } - /* Otherwise fall through to the next case */ + break; } #endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger funcction and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + case SQLITE_CONFIG_URI: { + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + default: { - if( !IdChar(*z) ){ - break; - } - for(i=1; IdChar(z[i]); i++){} - *tokenType = keywordCode((char*)z, i); - return i; + rc = SQLITE_ERROR; + break; } } - *tokenType = TK_ILLEGAL; - return 1; + va_end(ap); + return rc; } /* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. */ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ - int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ - void *pEngine; /* The LEMON-generated LALR(1) parser */ - int tokenType; /* type of the next token */ - int lastTokenParsed = -1; /* type of the previous token */ - u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ - sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ - - - mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - if( db->activeVdbeCnt==0 ){ - db->u1.isInterrupted = 0; +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ + void *pStart; + if( db->lookaside.nOut ){ + return SQLITE_BUSY; } - pParse->rc = SQLITE_OK; - pParse->zTail = zSql; - i = 0; - assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); - if( pEngine==0 ){ - db->mallocFailed = 1; - return SQLITE_NOMEM; + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); } - assert( pParse->pNewTable==0 ); - assert( pParse->pNewTrigger==0 ); - assert( pParse->nVar==0 ); - assert( pParse->nVarExpr==0 ); - assert( pParse->nVarExprAlloc==0 ); - assert( pParse->apVarExpr==0 ); - enableLookaside = db->lookaside.bEnabled; - if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; - while( !db->mallocFailed && zSql[i]!=0 ){ - assert( i>=0 ); - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ - pParse->rc = SQLITE_TOOBIG; + /* The size of a lookaside slot needs to be larger than a pointer + ** to be useful. + */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + }else{ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + pStart = pBuf; + } + db->lookaside.pStart = pStart; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = db->lookaside.pFree; + db->lookaside.pFree = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + db->lookaside.pEnd = p; + db->lookaside.bEnabled = 1; + db->lookaside.bMalloced = pBuf==0 ?1:0; + }else{ + db->lookaside.pEnd = 0; + db->lookaside.bEnabled = 0; + db->lookaside.bMalloced = 0; + } + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ + return db->mutex; +} + +/* +** Configuration settings for an individual database connection +*/ +SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); break; } - switch( tokenType ){ - case TK_SPACE: { - if( db->u1.isInterrupted ){ - sqlite3ErrorMsg(pParse, "interrupt"); - pParse->rc = SQLITE_INTERRUPT; - goto abort_parse; - } - break; - } - case TK_ILLEGAL: { - sqlite3DbFree(db, *pzErrMsg); - *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", - &pParse->sLastToken); - nErr++; - goto abort_parse; - } - case TK_SEMI: { - pParse->zTail = &zSql[i]; - /* Fall thru into the default case */ - } - default: { - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK ){ - goto abort_parse; + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; } - break; } + break; } } -abort_parse: - if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ - if( lastTokenParsed!=TK_SEMI ){ - sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - pParse->zTail = &zSql[i]; + va_end(ap); + return rc; +} + + +/* +** Return true if the buffer z[0..n-1] contains all spaces. +*/ +static int allSpaces(const char *z, int n){ + while( n>0 && z[n-1]==' ' ){ n--; } + return n==0; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +** +** If the padFlag argument is not NULL then space padding at the end +** of strings is ignored. This implements the RTRIM collation. +*/ +static int binCollFunc( + void *padFlag, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1sLastToken, pParse); } -#ifdef YYTRACKMAXSTACKDEPTH - sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, - sqlite3ParserStackPeak(pEngine) - ); -#endif /* YYDEBUG */ - sqlite3ParserFree(pEngine, sqlite3_free); - db->lookaside.bEnabled = enableLookaside; - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + return rc; +} + +/* +** Another built-in collating sequence: NOCASE. +** +** This collating sequence is intended to be used for "case independant +** comparison". SQLite's knowledge of upper and lower case equivalents +** extends only to the 26 characters used in the English language. +** +** At the moment there is only a UTF-8 implementation. +*/ +static int nocaseCollatingFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int r = sqlite3StrNICmp( + (const char *)pKey1, (const char *)pKey2, (nKey1rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); + return r; +} + +/* +** Return the ROWID of the most recent insert +*/ +SQLITE_API sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ + return db->lastRowid; +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +SQLITE_API int sqlite3_changes(sqlite3 *db){ + return db->nChange; +} + +/* +** Return the number of changes since the database handle was opened. +*/ +SQLITE_API int sqlite3_total_changes(sqlite3 *db){ + return db->nTotalChange; +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); - pParse->zErrMsg = 0; - nErr++; + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } } - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; +} + +/* +** Close an existing SQLite database +*/ +SQLITE_API int sqlite3_close(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + if( !db ){ + return SQLITE_OK; } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + + /* Force xDestroy calls on all virtual tables */ + sqlite3ResetInternalSchema(db, -1); + + /* If a transaction is open, the ResetInternalSchema() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* If there are any outstanding VMs, return SQLITE_BUSY. */ + if( db->pVdbe ){ + sqlite3Error(db, SQLITE_BUSY, + "unable to close due to unfinalised statements"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + assert( sqlite3SafetyCheckSickOrOk(db) ); + + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ){ + sqlite3Error(db, SQLITE_BUSY, + "unable to close due to unfinished backup operation"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + } + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; + } + } + } + sqlite3ResetInternalSchema(db, -1); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + for(j=0; jaFunc.a); j++){ + FuncDef *pNext, *pHash, *p; + for(p=db->aFunc.a[j]; p; p=pHash){ + pHash = p->pHash; + while( p ){ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + } + } + } + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } + } + sqlite3DbFree(db, pColl); } -#endif + sqlite3HashClear(&db->aCollSeq); #ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_free(pParse->apVtabLock); + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + sqlite3DbFree(db, pMod); + } + sqlite3HashClear(&db->aModule); #endif - if( !IN_DECLARE_VTAB ){ - /* If the pParse->declareVtab flag is set, do not delete any table - ** structure built up in pParse->pNewTable. The calling code (see vtab.c) - ** will take responsibility for freeing the Table structure. - */ - sqlite3DeleteTable(db, pParse->pNewTable); + sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ + if( db->pErr ){ + sqlite3ValueFree(db->pErr); } + sqlite3CloseExtensions(db); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); - sqlite3DbFree(db, pParse->apVarExpr); - sqlite3DbFree(db, pParse->aAlias); - while( pParse->pAinc ){ - AutoincInfo *p = pParse->pAinc; - pParse->pAinc = p->pNext; - sqlite3DbFree(db, p); - } - while( pParse->pZombieTab ){ - Table *p = pParse->pZombieTab; - pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(db, p); - } - if( nErr>0 && pParse->rc==SQLITE_OK ){ - pParse->rc = SQLITE_ERROR; + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); } - return nErr; + sqlite3_free(db); + return SQLITE_OK; } -/************** End of tokenize.c ********************************************/ -/************** Begin file complete.c ****************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** An tokenizer for SQL -** -** This file contains C code that implements the sqlite3_complete() API. -** This code used to be part of the tokenizer.c source file. But by -** separating it out, the code will be automatically omitted from -** static links that do not use it. +** Rollback all database files. */ -#ifndef SQLITE_OMIT_COMPLETE +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db){ + int i; + int inTrans = 0; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + for(i=0; inDb; i++){ + if( db->aDb[i].pBt ){ + if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ + inTrans = 1; + } + sqlite3BtreeRollback(db->aDb[i].pBt); + db->aDb[i].inTrans = 0; + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); -/* -** This is defined in tokenize.c. We just have to import the definition. -*/ -#ifndef SQLITE_AMALGAMATION -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) -#endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) -#endif -#endif /* SQLITE_AMALGAMATION */ + if( db->flags&SQLITE_InternChanges ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetInternalSchema(db, -1); + } + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; -/* -** Token types used by the sqlite3_complete() routine. See the header -** comments on that procedure for additional information. -*/ -#define tkSEMI 0 -#define tkWS 1 -#define tkOTHER 2 -#ifndef SQLITE_OMIT_TRIGGER -#define tkEXPLAIN 3 -#define tkCREATE 4 -#define tkTEMP 5 -#define tkTRIGGER 6 -#define tkEND 7 -#endif + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } +} /* -** Return TRUE if the given SQL string ends in a semicolon. -** -** Special handling is require for CREATE TRIGGER statements. -** Whenever the CREATE TRIGGER keywords are seen, the statement -** must end with ";END;". -** -** This implementation uses a state machine with 8 states: -** -** (0) INVALID We have not yet seen a non-whitespace character. -** -** (1) START At the beginning or end of an SQL statement. This routine -** returns 1 if it ends in the START state and 0 if it ends -** in any other state. -** -** (2) NORMAL We are in the middle of statement which ends with a single -** semicolon. -** -** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of -** a statement. -** -** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceeded by EXPLAIN and/or followed by -** TEMP or TEMPORARY -** -** (5) TRIGGER We are in the middle of a trigger definition that must be -** ended by a semicolon, the keyword END, and another semicolon. -** -** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at -** the end of a trigger definition. -** -** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger difinition. -** -** Transitions between states above are determined by tokens extracted -** from the input. The following tokens are significant: -** -** (0) tkSEMI A semicolon. -** (1) tkWS Whitespace. -** (2) tkOTHER Any other SQL token. -** (3) tkEXPLAIN The "explain" keyword. -** (4) tkCREATE The "create" keyword. -** (5) tkTEMP The "temp" or "temporary" keyword. -** (6) tkTRIGGER The "trigger" keyword. -** (7) tkEND The "end" keyword. -** -** Whitespace never causes a state transition and is always ignored. -** This means that a SQL string of all whitespace is invalid. -** -** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed -** to recognize the end of a trigger can be omitted. All we have to do -** is look for a semicolon that is not part of an string or comment. +** Return a static string that describes the kind of error specified in the +** argument. */ -SQLITE_API int sqlite3_complete(const char *zSql){ - u8 state = 0; /* Current state, using numbers defined in header comment */ - u8 token; /* Value of the next token */ - -#ifndef SQLITE_OMIT_TRIGGER - /* A complex statement machine used to detect the end of a CREATE TRIGGER - ** statement. This is the normal case. - */ - static const u8 trans[8][8] = { - /* Token: */ - /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, - /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, - /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, - /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, - /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, - /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, - /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, - }; -#else - /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simplier - */ - static const u8 trans[3][3] = { - /* Token: */ - /* State: ** SEMI WS OTHER */ - /* 0 INVALID: */ { 1, 0, 2, }, - /* 1 START: */ { 1, 1, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, }, +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error or missing database", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "callback requested query abort", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ "table contains no data", + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "library routine called out of sequence", + /* SQLITE_NOLFS */ "large file support is disabled", + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ "auxiliary database format error", + /* SQLITE_RANGE */ "bind or column index out of range", + /* SQLITE_NOTADB */ "file is encrypted or is not a database", }; -#endif /* SQLITE_OMIT_TRIGGER */ - - while( *zSql ){ - switch( *zSql ){ - case ';': { /* A semicolon */ - token = tkSEMI; - break; - } - case ' ': - case '\r': - case '\t': - case '\n': - case '\f': { /* White space is ignored */ - token = tkWS; - break; - } - case '/': { /* C-style comments */ - if( zSql[1]!='*' ){ - token = tkOTHER; - break; - } - zSql += 2; - while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } - if( zSql[0]==0 ) return 0; - zSql++; - token = tkWS; - break; - } - case '-': { /* SQL-style comments from "--" to end of line */ - if( zSql[1]!='-' ){ - token = tkOTHER; - break; - } - while( *zSql && *zSql!='\n' ){ zSql++; } - if( *zSql==0 ) return state==1; - token = tkWS; - break; - } - case '[': { /* Microsoft-style identifiers in [...] */ - zSql++; - while( *zSql && *zSql!=']' ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; - } - case '`': /* Grave-accent quoted symbols used by MySQL */ - case '"': /* single- and double-quoted strings */ - case '\'': { - int c = *zSql; - zSql++; - while( *zSql && *zSql!=c ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; - } - default: { -#ifdef SQLITE_EBCDIC - unsigned char c; -#endif - if( IdChar((u8)*zSql) ){ - /* Keywords and unquoted identifiers */ - int nId; - for(nId=1; IdChar(zSql[nId]); nId++){} -#ifdef SQLITE_OMIT_TRIGGER - token = tkOTHER; -#else - switch( *zSql ){ - case 'c': case 'C': { - if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ - token = tkCREATE; - }else{ - token = tkOTHER; - } - break; - } - case 't': case 'T': { - if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ - token = tkTRIGGER; - }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ - token = tkTEMP; - }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ - token = tkTEMP; - }else{ - token = tkOTHER; - } - break; - } - case 'e': case 'E': { - if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ - token = tkEND; - }else -#ifndef SQLITE_OMIT_EXPLAIN - if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ - token = tkEXPLAIN; - }else -#endif - { - token = tkOTHER; - } - break; - } - default: { - token = tkOTHER; - break; - } - } -#endif /* SQLITE_OMIT_TRIGGER */ - zSql += nId-1; - }else{ - /* Operators and special symbols */ - token = tkOTHER; - } - break; - } - } - state = trans[state][token]; - zSql++; + rc &= 0xff; + if( ALWAYS(rc>=0) && rc<(int)(sizeof(aMsg)/sizeof(aMsg[0])) && aMsg[rc]!=0 ){ + return aMsg[rc]; + }else{ + return "unknown error"; } - return state==1; } -#ifndef SQLITE_OMIT_UTF16 /* -** This routine is the same as the sqlite3_complete() routine described -** above, except that the parameter is required to be UTF-16 encoded, not -** UTF-8. +** This routine implements a busy callback that sleeps and tries +** again until a timeout value is reached. The timeout value is +** an integer number of milliseconds passed in as the first +** argument. */ -SQLITE_API int sqlite3_complete16(const void *zSql){ - sqlite3_value *pVal; - char const *zSql8; - int rc = SQLITE_NOMEM; +static int sqliteDefaultBusyCallback( + void *ptr, /* Database connection */ + int count /* Number of times table has been busy */ +){ +#if SQLITE_OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) + static const u8 delays[] = + { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; + static const u8 totals[] = + { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; +# define NDELAY ArraySize(delays) + sqlite3 *db = (sqlite3 *)ptr; + int timeout = db->busyTimeout; + int delay, prior; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zSql8 ){ - rc = sqlite3_complete(zSql8); + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; }else{ - rc = SQLITE_NOMEM; + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); } - sqlite3ValueFree(pVal); - return sqlite3ApiExit(0, rc); + if( prior + delay > timeout ){ + delay = timeout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + sqlite3 *db = (sqlite3 *)ptr; + int timeout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > timeout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif } -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_OMIT_COMPLETE */ -/************** End of complete.c ********************************************/ -/************** Begin file main.c ********************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Invoke the given busy handler. ** -************************************************************************* -** Main file for the SQLite library. The routines in this file -** implement the programmer interface to the library. Routines in -** other files are for internal use by SQLite and should not be -** accessed by users of the library. +** This routine is called when an operation failed with a lock. +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. */ +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; + rc = p->xFunc(p->pArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} -#ifdef SQLITE_ENABLE_FTS3 -/************** Include fts3.h in the middle of main.c ***********************/ -/************** Begin file fts3.h ********************************************/ /* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. */ +SQLITE_API int sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xFunc = xBusy; + db->busyHandler.pArg = pArg; + db->busyHandler.nBusy = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +SQLITE_API void sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} +#endif -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ -/************** End of fts3.h ************************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_RTREE -/************** Include rtree.h in the middle of main.c **********************/ -/************** Begin file rtree.h *******************************************/ /* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. */ +SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ + if( ms>0 ){ + db->busyTimeout = ms; + sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); +/* +** Cause any pending operation to stop at its earliest opportunity. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3 *db){ + db->u1.isInterrupted = 1; +} -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ -/************** End of rtree.h ***********************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_ICU -/************** Include sqliteicu.h in the middle of main.c ******************/ -/************** Begin file sqliteicu.h ***************************************/ /* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** ICU extension. All it does is declare the sqlite3IcuInit() interface. +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. */ +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + assert( sqlite3_mutex_held(db->mutex) ); + if( zFunctionName==0 || + (xFunc && (xFinal || xStep)) || + (!xFunc && (xFinal && !xStep)) || + (!xFunc && (!xFinal && xStep)) || + (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || + (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + return SQLITE_MISUSE_BKPT; + } + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, + pUserData, xFunc, xStep, xFinal, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, + pUserData, xFunc, xStep, xFinal, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + if( p && p->iPrefEnc==enc && p->nArg==nArg ){ + if( db->activeVdbeCnt ){ + sqlite3Error(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db); + } + } -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM; + } + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); -/************** End of sqliteicu.h *******************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; + p->flags = 0; + p->xFunc = xFunc; + p->xStep = xStep; + p->xFinalize = xFinal; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} -#ifndef SQLITE_AMALGAMATION -/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant -** contains the text of SQLITE_VERSION macro. +/* +** Create new user functions. */ -SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; -#endif +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} -/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. -*/ -SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } -/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a -** pointer to a string constant whose value is the same as the -** SQLITE_SOURCE_ID C preprocessor macro. -*/ -SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} -/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function -** returns an integer equal to SQLITE_VERSION_NUMBER. -*/ -SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif -/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled mutexing code omitted due to -** the SQLITE_THREADSAFE compile-time option being set to 0. -*/ -SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } -#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* -** If the following function pointer is not NULL and if -** SQLITE_ENABLE_IOTRACE is enabled, then messages describing -** I/O active are written using this function. These messages -** are intended for debugging activity only. +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. */ -SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*, ...) = 0; -#endif +SQLITE_API int sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int nName = sqlite3Strlen30(zName); + int rc; + sqlite3_mutex_enter(db->mutex); + if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); + } + rc = sqlite3ApiExit(db, SQLITE_OK); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#ifndef SQLITE_OMIT_TRACE /* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** temporary files. +** Register a trace function. The pArg from the previously registered trace +** is returned. ** -** See also the "PRAGMA temp_store_directory" SQL command. +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. */ -SQLITE_API char *sqlite3_temp_directory = 0; - +SQLITE_API void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} /* -** Initialize SQLite. -** -** This routine must be called to initialize the memory allocation, -** VFS, and mutex subsystems prior to doing any serious work with -** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT -** this routine will be called automatically by key routines such as -** sqlite3_open(). -** -** This routine is a no-op except on its very first call for the process, -** or for the first call after a call to sqlite3_shutdown. -** -** The first thread to call this routine runs the initialization to -** completion. If subsequent threads call this routine before the first -** thread has finished the initialization process, then the subsequent -** threads must block until the first thread finishes with the initialization. -** -** The first thread might call this routine recursively. Recursive -** calls to this routine should not block, of course. Otherwise the -** initialization process would never complete. -** -** Let X be the first thread to enter this routine. Let Y be some other -** thread. Then while the initial invocation of this routine by X is -** incomplete, it is required that: -** -** * Calls to this routine from Y must block until the outer-most -** call by X completes. +** Register a profile function. The pArg from the previously registered +** profile function is returned. ** -** * Recursive calls to this routine from thread X return immediately -** without blocking. +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. */ -SQLITE_API int sqlite3_initialize(void){ - sqlite3_mutex *pMaster; /* The main static mutex */ - int rc; /* Result code */ - -#ifdef SQLITE_OMIT_WSD - rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; - } -#endif - - /* If SQLite is already completely initialized, then this call - ** to sqlite3_initialize() should be a no-op. But the initialization - ** must be complete. So isInit must not be set until the very end - ** of this routine. - */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; - - /* Make sure the mutex subsystem is initialized. If unable to - ** initialize the mutex subsystem, return early with the error. - ** If the system is so sick that we are unable to allocate a mutex, - ** there is not much SQLite is going to be able to do. - ** - ** The mutex subsystem must take care of serializing its own - ** initialization. - */ - rc = sqlite3MutexInit(); - if( rc ) return rc; +SQLITE_API void *sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_TRACE */ - /* Initialize the malloc() system and the recursive pInitMutex mutex. - ** This operation is protected by the STATIC_MASTER mutex. Note that - ** MutexAlloc() is called for a static mutex prior to initializing the - ** malloc subsystem - this implies that the allocation of a static - ** mutex must not require support from the malloc subsystem. - */ - pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.isMutexInit = 1; - if( !sqlite3GlobalConfig.isMallocInit ){ - rc = sqlite3MallocInit(); - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isMallocInit = 1; - if( !sqlite3GlobalConfig.pInitMutex ){ - sqlite3GlobalConfig.pInitMutex = - sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM; - } - } - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.nRefInitMutex++; - } - sqlite3_mutex_leave(pMaster); +/*** EXPERIMENTAL *** +** +** Register a function to be invoked when a transaction comments. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +SQLITE_API void *sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} - /* If rc is not SQLITE_OK at this point, then either the malloc - ** subsystem could not be initialized or the system failed to allocate - ** the pInitMutex mutex. Return an error in either case. */ - if( rc!=SQLITE_OK ){ - return rc; - } +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} - /* Do the rest of the initialization under the recursive mutex so - ** that we will be able to handle recursive calls into - ** sqlite3_initialize(). The recursive calls normally come through - ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other - ** recursive calls might also be possible. - ** - ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls - ** to the xInit method, so the xInit method need not be threadsafe. - ** - ** The following mutex is what serializes access to the appdef pcache xInit - ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the - ** call to sqlite3PcacheInitialize(). - */ - sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); - if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - sqlite3GlobalConfig.inProgress = 1; - memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); - sqlite3RegisterGlobalFunctions(); - if( sqlite3GlobalConfig.isPCacheInit==0 ){ - rc = sqlite3PcacheInitialize(); - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isPCacheInit = 1; - rc = sqlite3OsInit(); - } - if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, - sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); - sqlite3GlobalConfig.isInit = 1; - } - sqlite3GlobalConfig.inProgress = 0; - } - sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. +*/ +SQLITE_API void *sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} - /* Go back under the static mutex and clean up the recursive - ** mutex to prevent a resource leak. - */ - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.nRefInitMutex--; - if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ - assert( sqlite3GlobalConfig.nRefInitMutex==0 ); - sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); - sqlite3GlobalConfig.pInitMutex = 0; +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); } - sqlite3_mutex_leave(pMaster); + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ - /* The following is just a sanity check to make sure SQLite has - ** been compiled correctly. It is important to run this code, but - ** we don't want to run it too often and soak up CPU cycles for no - ** reason. So we run it once during initialization. - */ -#ifndef NDEBUG -#ifndef SQLITE_OMIT_FLOATING_POINT - /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE_OK ){ - u64 x = (((u64)1)<<63)-1; - double y; - assert(sizeof(x)==8); - assert(sizeof(x)==sizeof(y)); - memcpy(&y, &x, 8); - assert( sqlite3IsNaN(y) ); +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); } #endif -#endif - - return rc; + return SQLITE_OK; } /* -** Undo the effects of sqlite3_initialize(). Must not be called while -** there are outstanding database connections or memory allocations or -** while any part of SQLite is otherwise in use in any thread. This -** routine is not threadsafe. But it is safe to invoke this routine -** on when SQLite is already shut down. If SQLite is already shut down -** when this routine is invoked, then this routine is a harmless no-op. +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. */ -SQLITE_API int sqlite3_shutdown(void){ - if( sqlite3GlobalConfig.isInit ){ - sqlite3_os_end(); - sqlite3_reset_auto_extension(); - sqlite3GlobalConfig.isInit = 0; - } - if( sqlite3GlobalConfig.isPCacheInit ){ - sqlite3PcacheShutdown(); - sqlite3GlobalConfig.isPCacheInit = 0; - } - if( sqlite3GlobalConfig.isMallocInit ){ - sqlite3MallocEnd(); - sqlite3GlobalConfig.isMallocInit = 0; - } - if( sqlite3GlobalConfig.isMutexInit ){ - sqlite3MutexEnd(); - sqlite3GlobalConfig.isMutexInit = 0; - } - - return SQLITE_OK; +SQLITE_API void *sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; +#endif } /* -** This API allows applications to modify the global configuration of -** the SQLite library at run-time. -** -** This routine should only be called when there are no outstanding -** database connections or memory allocations. This routine is not -** threadsafe. Failure to heed these warnings can lead to unpredictable -** behavior. +** Checkpoint database zDb. */ -SQLITE_API int sqlite3_config(int op, ...){ - va_list ap; - int rc = SQLITE_OK; +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; - va_start(ap, op); - switch( op ){ + assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); + assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + return SQLITE_MISUSE; + } - /* Mutex configuration options are only available in a threadsafe - ** compile. - */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 - case SQLITE_CONFIG_SINGLETHREAD: { - /* Disable all mutexing */ - sqlite3GlobalConfig.bCoreMutex = 0; - sqlite3GlobalConfig.bFullMutex = 0; - break; - } - case SQLITE_CONFIG_MULTITHREAD: { - /* Disable mutexing of database connections */ - /* Enable mutexing of core data structures */ - sqlite3GlobalConfig.bCoreMutex = 1; - sqlite3GlobalConfig.bFullMutex = 0; - break; - } - case SQLITE_CONFIG_SERIALIZED: { - /* Enable all mutexing */ - sqlite3GlobalConfig.bCoreMutex = 1; - sqlite3GlobalConfig.bFullMutex = 1; - break; - } - case SQLITE_CONFIG_MUTEX: { - /* Specify an alternative mutex implementation */ - sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); - break; - } - case SQLITE_CONFIG_GETMUTEX: { - /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; - break; - } + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc, 0); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; #endif +} - case SQLITE_CONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ - sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); - break; - } - case SQLITE_CONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ - if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); - *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; - break; - } - case SQLITE_CONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_SCRATCH: { - /* Designate a buffer for scratch memory space */ - sqlite3GlobalConfig.pScratch = va_arg(ap, void*); - sqlite3GlobalConfig.szScratch = va_arg(ap, int); - sqlite3GlobalConfig.nScratch = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PAGECACHE: { - /* Designate a buffer for page cache memory space */ - sqlite3GlobalConfig.pPage = va_arg(ap, void*); - sqlite3GlobalConfig.szPage = va_arg(ap, int); - sqlite3GlobalConfig.nPage = va_arg(ap, int); - break; - } +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); +} - case SQLITE_CONFIG_PCACHE: { - /* Specify an alternative page cache implementation */ - sqlite3GlobalConfig.pcache = *va_arg(ap, sqlite3_pcache_methods*); - break; - } +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ - case SQLITE_CONFIG_GETPCACHE: { - if( sqlite3GlobalConfig.pcache.xInit==0 ){ - sqlite3PCacheSetDefault(); + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; } - *va_arg(ap, sqlite3_pcache_methods*) = sqlite3GlobalConfig.pcache; - break; } + } -#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) - case SQLITE_CONFIG_HEAP: { - /* Designate a buffer for heap memory space */ - sqlite3GlobalConfig.pHeap = va_arg(ap, void*); - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - sqlite3GlobalConfig.mnReq = va_arg(ap, int); + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ - if( sqlite3GlobalConfig.pHeap==0 ){ - /* If the heap pointer is NULL, then restore the malloc implementation - ** back to NULL pointers too. This will cause the malloc to go - ** back to its default implementation when sqlite3_initialize() is - ** run. - */ - memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); - }else{ - /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. If neither ENABLE_MEMSYS3 nor - ** ENABLE_MEMSYS5 is defined, return an error. - */ -#ifdef SQLITE_ENABLE_MEMSYS3 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); #endif -#ifdef SQLITE_ENABLE_MEMSYS5 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); #endif - } - break; - } +#if SQLITE_TEMP_STORE==3 + return 1; #endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + return 0; +#endif +} - case SQLITE_CONFIG_LOOKASIDE: { - sqlite3GlobalConfig.szLookaside = va_arg(ap, int); - sqlite3GlobalConfig.nLookaside = va_arg(ap, int); - break; - } - - /* Record a pointer to the logger funcction and its first argument. - ** The default is NULL. Logging is disabled if the function pointer is - ** NULL. - */ - case SQLITE_CONFIG_LOG: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); - */ - typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); - break; +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM); + }else{ + z = (char*)sqlite3_value_text(db->pErr); + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); } + } + sqlite3_mutex_leave(db->mutex); + return z; +} - default: { - rc = SQLITE_ERROR; - break; +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', + 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', + 'c', 'a', 'l', 'l', 'e', 'd', ' ', + 'o', 'u', 't', ' ', + 'o', 'f', ' ', + 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ValueSetStr(db->pErr, -1, sqlite3ErrStr(db->errCode), + SQLITE_UTF8, SQLITE_STATIC); + z = sqlite3_value_text16(db->pErr); } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + db->mallocFailed = 0; } - va_end(ap); - return rc; + sqlite3_mutex_leave(db->mutex); + return z; } +#endif /* SQLITE_OMIT_UTF16 */ /* -** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. -** If lookaside is already active, return SQLITE_BUSY. -** -** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ - void *pStart; - if( db->lookaside.nOut ){ - return SQLITE_BUSY; +SQLITE_API int sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; } - /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for - ** both at the same time. - */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; } - /* The size of a lookaside slot needs to be larger than a pointer - ** to be useful. + return db->errCode & db->errMask; +} +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode; +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + u8 collType, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + int nName = sqlite3Strlen30(zName); + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. */ - if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ - sz = 0; - pStart = 0; - }else if( pBuf==0 ){ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ - sqlite3EndBenignMalloc(); - }else{ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - pStart = pBuf; + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; } - db->lookaside.pStart = pStart; - db->lookaside.pFree = 0; - db->lookaside.sz = (u16)sz; - if( pStart ){ - int i; - LookasideSlot *p; - assert( sz > (int)sizeof(LookasideSlot*) ); - p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pFree; - db->lookaside.pFree = p; - p = (LookasideSlot*)&((u8*)p)[sz]; + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->activeVdbeCnt ){ + sqlite3Error(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } } - db->lookaside.pEnd = p; - db->lookaside.bEnabled = 1; - db->lookaside.bMalloced = pBuf==0 ?1:0; - }else{ - db->lookaside.pEnd = 0; - db->lookaside.bEnabled = 0; - db->lookaside.bMalloced = 0; } + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + pColl->type = collType; + sqlite3Error(db, SQLITE_OK, 0); return SQLITE_OK; } -/* -** Return the mutex associated with a database connection. -*/ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ - return db->mutex; -} /* -** Configuration settings for an individual database connection +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. */ -SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ - va_list ap; - int rc; - va_start(ap, op); - switch( op ){ - case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); /* IMP: R-21112-12275 */ - int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ - int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ - rc = setupLookaside(db, pBuf, sz, cnt); - break; - } - default: { - rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ - break; - } - } - va_end(ap); - return rc; -} - +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, + SQLITE_MAX_TRIGGER_DEPTH, +}; /* -** Return true if the buffer z[0..n-1] contains all spaces. +** Make sure the hard limits are set to reasonable values */ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; -} +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>62 +# error SQLITE_MAX_ATTACHED must be between 0 and 62 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif + /* -** This is the default collating function named "BINARY" which is always -** available. +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. ** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. */ -static int binCollFunc( - void *padFlag, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 -){ - int rc, n; - n = nKey1=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ } + db->aLimit[limitId] = newLimit; } - return rc; + return oldLimit; /* IMP: R-53341-35419 */ } /* -** Another built-in collating sequence: NOCASE. +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. ** -** This collating sequence is intended to be used for "case independant -** comparison". SQLite's knowledge of upper and lower case equivalents -** extends only to the 26 characters used in the English language. +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. ** -** At the moment there is only a UTF-8 implementation. +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). */ -static int nocaseCollatingFunc( - void *NotUsed, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ ){ - int r = sqlite3StrNICmp( - (const char *)pKey1, (const char *)pKey2, (nKey1lastRowid; -} + assert( *pzErrMsg==0 ); -/* -** Return the number of changes in the most recent call to sqlite3_exec(). -*/ -SQLITE_API int sqlite3_changes(sqlite3 *db){ - return db->nChange; -} + if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) + && nUri>=5 && memcmp(zUri, "file:", 5)==0 + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + int nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iInnTotalChange; -} + /* Copy the filename and any query parameters into the zFile buffer. + ** Decode %HH escape codes along the way. + ** + ** Within this loop, variable eState may be set to 0, 1 or 2, depending + ** on the parsing context. As follows: + ** + ** 0: Parsing file-name. + ** 1: Parsing name section of a name=value query parameter. + ** 2: Parsing value section of a name=value query parameter. + */ + eState = 0; + while( (c = zUri[iIn])!=0 && c!='#' ){ + iIn++; + if( c=='%' + && sqlite3Isxdigit(zUri[iIn]) + && sqlite3Isxdigit(zUri[iIn+1]) + ){ + int octet = (sqlite3HexToInt(zUri[iIn++]) << 4); + octet += sqlite3HexToInt(zUri[iIn++]); + + assert( octet>=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; -/* -** Close all open savepoints. This function only manipulates fields of the -** database handle object, it does not close any savepoints that may be open -** at the b-tree/pager level. -*/ -SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ - while( db->pSavepoint ){ - Savepoint *pTmp = db->pSavepoint; - db->pSavepoint = pTmp->pNext; - sqlite3DbFree(db, pTmp); - } - db->nSavepoint = 0; - db->nStatement = 0; - db->isTransactionSavepoint = 0; -} + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { 0, 0 } + }; -/* -** Invoke the destructor function associated with FuncDef p, if any. Except, -** if this is not the last copy of the function, do not invoke it. Multiple -** copies of a single function are created when create_function() is called -** with SQLITE_ANY as the encoding. -*/ -static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->pDestructor; - if( pDestructor ){ - pDestructor->nRef--; - if( pDestructor->nRef==0 ){ - pDestructor->xDestroy(pDestructor->pUserData); - sqlite3DbFree(db, pDestructor); + mask = SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( mode>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; } + + }else{ + zFile = sqlite3_malloc(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; } + *pFlags = flags; + *pzFile = zFile; + return rc; } + /* -** Close an existing SQLite database +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. */ -SQLITE_API int sqlite3_close(sqlite3 *db){ - HashElem *i; /* Hash table iterator */ - int j; +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ - if( !db ){ - return SQLITE_OK; + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } } sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; - sqlite3ResetInternalSchema(db, 0); + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->autoCommit = 1; + db->nextAutovac = -1; + db->nextPagesize = 0; + db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif - /* If a transaction is open, the ResetInternalSchema() call above - ** will not have called the xDisconnect() method on any virtual - ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() - ** call will do so. We need to do this before the check for active - ** SQL statements below, as the v-table implementation may be storing - ** some prepared statements internally. + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. */ - sqlite3VtabRollback(db); + createCollation(db, "BINARY", SQLITE_UTF8, SQLITE_COLL_BINARY, 0, + binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, SQLITE_COLL_BINARY, 0, + binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, SQLITE_COLL_BINARY, 0, + binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, SQLITE_COLL_USER, (void*)1, + binCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + assert( db->pDfltColl!=0 ); - /* If there are any outstanding VMs, return SQLITE_BUSY. */ - if( db->pVdbe ){ - sqlite3Error(db, SQLITE_BUSY, - "unable to close due to unfinalised statements"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; + /* Also add a UTF-8 case-insensitive collation sequence. */ + createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, + nocaseCollatingFunc, 0); + + /* Parse the filename/URI argument. */ + db->openFlags = flags; + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; } - assert( sqlite3SafetyCheckSickOrOk(db) ); - for(j=0; jnDb; j++){ - Btree *pBt = db->aDb[j].pBt; - if( pBt && sqlite3BtreeIsInBackup(pBt) ){ - sqlite3Error(db, SQLITE_BUSY, - "unable to close due to unfinished backup operation"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; } + sqlite3Error(db, rc, 0); + goto opendb_out; } + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - /* Free any outstanding Savepoint structures. */ - sqlite3CloseSavepoints(db); - for(j=0; jnDb; j++){ - struct Db *pDb = &db->aDb[j]; - if( pDb->pBt ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - if( j!=1 ){ - pDb->pSchema = 0; - } - } + /* The default safety_level for the main database is 'full'; for the temp + ** database it is 'NONE'. This matches the pager layer defaults. + */ + db->aDb[0].zName = "main"; + db->aDb[0].safety_level = 3; + db->aDb[1].zName = "temp"; + db->aDb[1].safety_level = 1; + + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; } - sqlite3ResetInternalSchema(db, 0); - /* Tell the code in notify.c that the connection no longer holds any - ** locks and does not require any further unlock-notify callbacks. + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. */ - sqlite3ConnectionClosed(db); + sqlite3Error(db, SQLITE_OK, 0); + sqlite3RegisterBuiltinFunctions(db); - assert( db->nDb<=2 ); - assert( db->aDb==db->aDbStatic ); - for(j=0; jaFunc.a); j++){ - FuncDef *pNext, *pHash, *p; - for(p=db->aFunc.a[j]; p; p=pHash){ - pHash = p->pHash; - while( p ){ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - } - } - } - for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ - CollSeq *pColl = (CollSeq *)sqliteHashData(i); - /* Invoke any destructors registered for collation sequence user data. */ - for(j=0; j<3; j++){ - if( pColl[j].xDel ){ - pColl[j].xDel(pColl[j].pUser); - } - } - sqlite3DbFree(db, pColl); - } - sqlite3HashClear(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ - Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); - } - sqlite3DbFree(db, pMod); + /* Load automatic extensions - extensions that have been registered + ** using the sqlite3_automatic_extension() API. + */ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; } - sqlite3HashClear(&db->aModule); -#endif - sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ - if( db->pErr ){ - sqlite3ValueFree(db->pErr); +#ifdef SQLITE_ENABLE_FTS1 + if( !db->mallocFailed ){ + extern int sqlite3Fts1Init(sqlite3*); + rc = sqlite3Fts1Init(db); } - sqlite3CloseExtensions(db); +#endif - db->magic = SQLITE_MAGIC_ERROR; +#ifdef SQLITE_ENABLE_FTS2 + if( !db->mallocFailed && rc==SQLITE_OK ){ + extern int sqlite3Fts2Init(sqlite3*); + rc = sqlite3Fts2Init(db); + } +#endif - /* The temp-database schema is allocated differently from the other schema - ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). - ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database - ** structure? - */ - sqlite3DbFree(db, db->aDb[1].pSchema); - sqlite3_mutex_leave(db->mutex); - db->magic = SQLITE_MAGIC_CLOSED; - sqlite3_mutex_free(db->mutex); - assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); +#ifdef SQLITE_ENABLE_FTS3 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts3Init(db); } - sqlite3_free(db); - return SQLITE_OK; -} +#endif -/* -** Rollback all database files. -*/ -SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db){ - int i; - int inTrans = 0; - assert( sqlite3_mutex_held(db->mutex) ); - sqlite3BeginBenignMalloc(); - for(i=0; inDb; i++){ - if( db->aDb[i].pBt ){ - if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ - inTrans = 1; - } - sqlite3BtreeRollback(db->aDb[i].pBt); - db->aDb[i].inTrans = 0; - } +#ifdef SQLITE_ENABLE_ICU + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3IcuInit(db); } - sqlite3VtabRollback(db); - sqlite3EndBenignMalloc(); +#endif - if( db->flags&SQLITE_InternChanges ){ - sqlite3ExpirePreparedStatements(db); - sqlite3ResetInternalSchema(db, 0); +#ifdef SQLITE_ENABLE_RTREE + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3RtreeInit(db); } +#endif - /* Any deferred constraint violations have now been resolved. */ - db->nDeferredCons = 0; + sqlite3Error(db, rc, 0); - /* If one has been configured, invoke the rollback-hook callback */ - if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ - db->xRollbackCallback(db->pRollbackArg); + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + sqlite3_free(zOpen); + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; } + *ppDb = db; + return sqlite3ApiExit(0, rc); } /* -** Return a static string that describes the kind of error specified in the -** argument. +** Open a new database handle. */ -SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ - static const char* const aMsg[] = { - /* SQLITE_OK */ "not an error", - /* SQLITE_ERROR */ "SQL logic error or missing database", - /* SQLITE_INTERNAL */ 0, - /* SQLITE_PERM */ "access permission denied", - /* SQLITE_ABORT */ "callback requested query abort", - /* SQLITE_BUSY */ "database is locked", - /* SQLITE_LOCKED */ "database table is locked", - /* SQLITE_NOMEM */ "out of memory", - /* SQLITE_READONLY */ "attempt to write a readonly database", - /* SQLITE_INTERRUPT */ "interrupted", - /* SQLITE_IOERR */ "disk I/O error", - /* SQLITE_CORRUPT */ "database disk image is malformed", - /* SQLITE_NOTFOUND */ 0, - /* SQLITE_FULL */ "database or disk is full", - /* SQLITE_CANTOPEN */ "unable to open database file", - /* SQLITE_PROTOCOL */ "locking protocol", - /* SQLITE_EMPTY */ "table contains no data", - /* SQLITE_SCHEMA */ "database schema has changed", - /* SQLITE_TOOBIG */ "string or blob too big", - /* SQLITE_CONSTRAINT */ "constraint failed", - /* SQLITE_MISMATCH */ "datatype mismatch", - /* SQLITE_MISUSE */ "library routine called out of sequence", - /* SQLITE_NOLFS */ "large file support is disabled", - /* SQLITE_AUTH */ "authorization denied", - /* SQLITE_FORMAT */ "auxiliary database format error", - /* SQLITE_RANGE */ "bind or column index out of range", - /* SQLITE_NOTADB */ "file is encrypted or is not a database", - }; - rc &= 0xff; - if( ALWAYS(rc>=0) && rc<(int)(sizeof(aMsg)/sizeof(aMsg[0])) && aMsg[rc]!=0 ){ - return aMsg[rc]; - }else{ - return "unknown error"; - } +SQLITE_API int sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); } +#ifndef SQLITE_OMIT_UTF16 /* -** This routine implements a busy callback that sleeps and tries -** again until a timeout value is reached. The timeout value is -** an integer number of milliseconds passed in as the first -** argument. +** Open a new database handle. */ -static int sqliteDefaultBusyCallback( - void *ptr, /* Database connection */ - int count /* Number of times table has been busy */ +SQLITE_API int sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb ){ -#if SQLITE_OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) - static const u8 delays[] = - { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; - static const u8 totals[] = - { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; -# define NDELAY (sizeof(delays)/sizeof(delays[0])) - sqlite3 *db = (sqlite3 *)ptr; - int timeout = db->busyTimeout; - int delay, prior; + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; - assert( count>=0 ); - if( count < NDELAY ){ - delay = delays[count]; - prior = totals[count]; + assert( zFilename ); + assert( ppDb ); + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + ENC(*ppDb) = SQLITE_UTF16NATIVE; + } }else{ - delay = delays[NDELAY-1]; - prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); - } - if( prior + delay > timeout ){ - delay = timeout - prior; - if( delay<=0 ) return 0; - } - sqlite3OsSleep(db->pVfs, delay*1000); - return 1; -#else - sqlite3 *db = (sqlite3 *)ptr; - int timeout = ((sqlite3 *)ptr)->busyTimeout; - if( (count+1)*1000 > timeout ){ - return 0; + rc = SQLITE_NOMEM; } - sqlite3OsSleep(db->pVfs, 1000000); - return 1; -#endif + sqlite3ValueFree(pVal); + + return sqlite3ApiExit(0, rc); } +#endif /* SQLITE_OMIT_UTF16 */ /* -** Invoke the given busy handler. -** -** This routine is called when an operation failed with a lock. -** If this routine returns non-zero, the lock is retried. If it -** returns 0, the operation aborts with an SQLITE_BUSY error. +** Register a new collation sequence with the database handle db. */ -SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ +SQLITE_API int sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ int rc; - if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; - rc = p->xFunc(p->pArg, p->nBusy); - if( rc==0 ){ - p->nBusy = -1; - }else{ - p->nBusy++; - } - return rc; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* -** This routine sets the busy callback for an Sqlite database to the -** given callback function with the given argument. +** Register a new collation sequence with the database handle db. */ -SQLITE_API int sqlite3_busy_handler( - sqlite3 *db, - int (*xBusy)(void*,int), - void *pArg +SQLITE_API int sqlite3_create_collation_v2( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) ){ + int rc; sqlite3_mutex_enter(db->mutex); - db->busyHandler.xFunc = xBusy; - db->busyHandler.pArg = pArg; - db->busyHandler.nBusy = 0; + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; + return rc; } -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +#ifndef SQLITE_OMIT_UTF16 /* -** This routine sets the progress callback for an Sqlite database to the -** given callback function with the given argument. The progress callback will -** be invoked every nOps opcodes. +** Register a new collation sequence with the database handle db. */ -SQLITE_API void sqlite3_progress_handler( - sqlite3 *db, - int nOps, - int (*xProgress)(void*), - void *pArg +SQLITE_API int sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) ){ + int rc = SQLITE_OK; + char *zName8; sqlite3_mutex_enter(db->mutex); - if( nOps>0 ){ - db->xProgress = xProgress; - db->nProgressOps = nOps; - db->pProgressArg = pArg; - }else{ - db->xProgress = 0; - db->nProgressOps = 0; - db->pProgressArg = 0; + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); } + rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); + return rc; } -#endif - +#endif /* SQLITE_OMIT_UTF16 */ /* -** This routine installs a default busy handler that waits for the -** specified number of milliseconds before returning 0. +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. */ -SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ - if( ms>0 ){ - db->busyTimeout = ms; - sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); - }else{ - sqlite3_busy_handler(db, 0, 0); - } +SQLITE_API int sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } +#ifndef SQLITE_OMIT_UTF16 /* -** Cause any pending operation to stop at its earliest opportunity. +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. */ -SQLITE_API void sqlite3_interrupt(sqlite3 *db){ - db->u1.isInterrupted = 1; +SQLITE_API int sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } +#endif /* SQLITE_OMIT_UTF16 */ - +#ifndef SQLITE_OMIT_DEPRECATED /* -** This function is exactly the same as sqlite3_create_function(), except -** that it is designed to be called by internal code. The difference is -** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. */ -SQLITE_PRIVATE int sqlite3CreateFunc( - sqlite3 *db, - const char *zFunctionName, - int nArg, - int enc, - void *pUserData, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - FuncDestructor *pDestructor -){ - FuncDef *p; - int nName; - - assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xFunc && (xFinal || xStep)) || - (!xFunc && (xFinal && !xStep)) || - (!xFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ - return SQLITE_MISUSE_BKPT; - } - -#ifndef SQLITE_OMIT_UTF16 - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - ** - ** If SQLITE_ANY is specified, add three versions of the function - ** to the hash table. - */ - if( enc==SQLITE_UTF16 ){ - enc = SQLITE_UTF16NATIVE; - }else if( enc==SQLITE_ANY ){ - int rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, - pUserData, xFunc, xStep, xFinal, pDestructor); - if( rc==SQLITE_OK ){ - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, - pUserData, xFunc, xStep, xFinal, pDestructor); - } - if( rc!=SQLITE_OK ){ - return rc; - } - enc = SQLITE_UTF16BE; - } -#else - enc = SQLITE_UTF8; +SQLITE_API int sqlite3_global_recover(void){ + return SQLITE_OK; +} #endif - - /* Check if an existing function is being overridden or deleted. If so, - ** and there are active VMs, then return SQLITE_BUSY. If a function - ** is being overridden/deleted but there are no active VMs, allow the - ** operation to continue but invalidate all precompiled statements. - */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); - if( p && p->iPrefEnc==enc && p->nArg==nArg ){ - if( db->activeVdbeCnt ){ - sqlite3Error(db, SQLITE_BUSY, - "unable to delete/modify user-function due to active statements"); - assert( !db->mallocFailed ); - return SQLITE_BUSY; - }else{ - sqlite3ExpirePreparedStatements(db); - } - } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); - assert(p || db->mallocFailed); - if( !p ){ - return SQLITE_NOMEM; - } +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +** +******* THIS IS AN EXPERIMENTAL API AND IS SUBJECT TO CHANGE ****** +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ + return db->autoCommit; +} - /* If an older version of the function with a configured destructor is - ** being replaced invoke the destructor function here. */ - functionDestroy(db, p); +/* +** The following routines are subtitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** constants. They server two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. +*/ +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CORRUPT, + "database corruption at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CORRUPT; +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_MISUSE, + "misuse at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_MISUSE; +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CANTOPEN, + "cannot open file at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CANTOPEN; +} - if( pDestructor ){ - pDestructor->nRef++; - } - p->pDestructor = pDestructor; - p->flags = 0; - p->xFunc = xFunc; - p->xStep = xStep; - p->xFinalize = xFinal; - p->pUserData = pUserData; - p->nArg = (u16)nArg; - return SQLITE_OK; + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. +*/ +SQLITE_API void sqlite3_thread_cleanup(void){ } +#endif /* -** Create new user functions. +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. */ -SQLITE_API int sqlite3_create_function( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) +#ifdef SQLITE_ENABLE_COLUMN_METADATA +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ ){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, - xFinal, 0); -} + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol; -SQLITE_API int sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) -){ - int rc = SQLITE_ERROR; - FuncDestructor *pArg = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + /* Ensure the database schema has been loaded */ sqlite3_mutex_enter(db->mutex); - if( xDestroy ){ - pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); - if( !pArg ){ - xDestroy(p); - goto out; + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } + + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || pTab->pSelect ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + if( iCol>=0 ){ + pCol = &pTab->aCol[iCol]; + } + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + pTab = 0; + goto error_out; } - pArg->xDestroy = xDestroy; - pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); - if( pArg && pArg->nRef==0 ){ - assert( rc!=SQLITE_OK ); - xDestroy(p); - sqlite3DbFree(db, pArg); + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = pCol->zType; + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = pCol->isPrimKey!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = "BINARY"; } - out: - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} +error_out: + sqlite3BtreeLeaveAll(db); -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_create_function16( - sqlite3 *db, - const void *zFunctionName, - int nArg, - int eTextRep, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value**), - void (*xStep)(sqlite3_context*,int,sqlite3_value**), - void (*xFinal)(sqlite3_context*) -){ - int rc; - char *zFunc8; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); - sqlite3DbFree(db, zFunc8); + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. + */ + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } #endif - /* -** Declare that a function has been overloaded by a virtual table. -** -** If the function already exists as a regular global function, then -** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. -** -** When virtual tables intend to provide an overloaded function, they -** should call this routine to make sure the global function exists. -** A global function must exist in order for name resolution to work -** properly. +** Sleep for a little while. Return the amount of time slept. */ -SQLITE_API int sqlite3_overload_function( - sqlite3 *db, - const char *zName, - int nArg -){ - int nName = sqlite3Strlen30(zName); +SQLITE_API int sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; int rc; - sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ - sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); - } - rc = sqlite3ApiExit(db, SQLITE_OK); - sqlite3_mutex_leave(db->mutex); + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); return rc; } -#ifndef SQLITE_OMIT_TRACE -/* -** Register a trace function. The pArg from the previously registered trace -** is returned. -** -** A NULL trace function means that no tracing is executes. A non-NULL -** trace is a pointer to a function that is invoked at the start of each -** SQL statement. -*/ -SQLITE_API void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ - void *pOld; - sqlite3_mutex_enter(db->mutex); - pOld = db->pTraceArg; - db->xTrace = xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; -} /* -** Register a profile function. The pArg from the previously registered -** profile function is returned. -** -** A NULL profile function means that no profiling is executes. A non-NULL -** profile is a pointer to a function that is invoked at the conclusion of -** each SQL statement that is run. -*/ -SQLITE_API void *sqlite3_profile( - sqlite3 *db, - void (*xProfile)(void*,const char*,sqlite_uint64), - void *pArg -){ - void *pOld; - sqlite3_mutex_enter(db->mutex); - pOld = db->pProfileArg; - db->xProfile = xProfile; - db->pProfileArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; -} -#endif /* SQLITE_OMIT_TRACE */ - -/*** EXPERIMENTAL *** -** -** Register a function to be invoked when a transaction comments. -** If the invoked function returns non-zero, then the commit becomes a -** rollback. +** Enable or disable the extended result codes. */ -SQLITE_API void *sqlite3_commit_hook( - sqlite3 *db, /* Attach the hook to this database */ - int (*xCallback)(void*), /* Function to invoke on each commit */ - void *pArg /* Argument to the function */ -){ - void *pOld; +SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); - pOld = db->pCommitArg; - db->xCommitCallback = xCallback; - db->pCommitArg = pArg; + db->errMask = onoff ? 0xffffffff : 0xff; sqlite3_mutex_leave(db->mutex); - return pOld; + return SQLITE_OK; } /* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. +** Invoke the xFileControl method on a particular database. */ -SQLITE_API void *sqlite3_update_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), - void *pArg /* Argument to the function */ -){ - void *pRet; +SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + int iDb; sqlite3_mutex_enter(db->mutex); - pRet = db->pUpdateArg; - db->xUpdateCallback = xCallback; - db->pUpdateArg = pArg; + if( zDbName==0 ){ + iDb = 0; + }else{ + for(iDb=0; iDbnDb; iDb++){ + if( strcmp(db->aDb[iDb].zName, zDbName)==0 ) break; + } + } + if( iDbnDb ){ + Btree *pBtree = db->aDb[iDb].pBt; + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; + } + sqlite3BtreeLeave(pBtree); + } + } sqlite3_mutex_leave(db->mutex); - return pRet; + return rc; } /* -** Register a callback to be invoked each time a transaction is rolled -** back by this database connection. +** Interface to the testing logic. */ -SQLITE_API void *sqlite3_rollback_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*), /* Callback function */ - void *pArg /* Argument to the function */ -){ - void *pRet; - sqlite3_mutex_enter(db->mutex); - pRet = db->pRollbackArg; - db->xRollbackCallback = xCallback; - db->pRollbackArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -} +SQLITE_API int sqlite3_test_control(int op, ...){ + int rc = 0; +#ifndef SQLITE_OMIT_BUILTIN_TEST + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* + ** Reset the PRNG back to its uninitialized state. The next call + ** to sqlite3_randomness() will reseed the PRNG using a single call + ** to the xRandomness method of the default VFS. + */ + case SQLITE_TESTCTRL_PRNG_RESET: { + sqlite3PrngResetState(); + break; + } + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** dileterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } + + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X). + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = ALWAYS(x); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) + ** + ** Set the nReserve size to N for the main database on the database + ** connection db. + */ + case SQLITE_TESTCTRL_RESERVE: { + sqlite3 *db = va_arg(ap, sqlite3*); + int x = va_arg(ap,int); + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); + sqlite3_mutex_leave(db->mutex); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + int x = va_arg(ap,int); + db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + break; + } + +#ifdef SQLITE_N_KEYWORD + /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + ** + ** If zWord is a keyword recognized by the parser, then return the + ** number of keywords. Or if zWord is not a keyword, return 0. + ** + ** This test feature is only available in the amalgamation since + ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite + ** is built using separate source files. + */ + case SQLITE_TESTCTRL_ISKEYWORD: { + const char *zWord = va_arg(ap, const char*); + int n = sqlite3Strlen30(zWord); + rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; + break; + } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_PGHDRSZ) + ** + ** Return the size of a pcache header in bytes. + */ + case SQLITE_TESTCTRL_PGHDRSZ: { + rc = sizeof(PgHdr); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } -#ifndef SQLITE_OMIT_WAL -/* -** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). -** Invoke sqlite3_wal_checkpoint if the number of frames in the log file -** is greater than sqlite3.pWalArg cast to an integer (the value configured by -** wal_autocheckpoint()). -*/ -SQLITE_PRIVATE int sqlite3WalDefaultHook( - void *pClientData, /* Argument */ - sqlite3 *db, /* Connection */ - const char *zDb, /* Database */ - int nFrame /* Size of WAL */ -){ - if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ - sqlite3BeginBenignMalloc(); - sqlite3_wal_checkpoint(db, zDb); - sqlite3EndBenignMalloc(); } - return SQLITE_OK; + va_end(ap); +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + return rc; } -#endif /* SQLITE_OMIT_WAL */ /* -** Configure an sqlite3_wal_hook() callback to automatically checkpoint -** a database after committing a transaction if there are nFrame or -** more frames in the log file. Passing zero or a negative value as the -** nFrame parameter disables automatic checkpoints entirely. +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. ** -** The callback registered by this function replaces any existing callback -** registered using sqlite3_wal_hook(). Likewise, registering a callback -** using sqlite3_wal_hook() disables the automatic checkpoint mechanism -** configured by this function. +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. */ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(nFrame); -#else - if( nFrame>0 ){ - sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); - }else{ - sqlite3_wal_hook(db, 0, 0); +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; } -#endif - return SQLITE_OK; + return 0; } +/************** End of main.c ************************************************/ +/************** Begin file notify.c ******************************************/ /* -** Register a callback to be invoked each time a transaction is written -** into the write-ahead-log by this database connection. +** 2009 March 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the sqlite3_unlock_notify() +** API method and its associated functionality. */ -SQLITE_API void *sqlite3_wal_hook( - sqlite3 *db, /* Attach the hook to this db handle */ - int(*xCallback)(void *, sqlite3*, const char*, int), - void *pArg /* First argument passed to xCallback() */ -){ -#ifndef SQLITE_OMIT_WAL - void *pRet; - sqlite3_mutex_enter(db->mutex); - pRet = db->pWalArg; - db->xWalCallback = xCallback; - db->pWalArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -#else - return 0; -#endif -} +/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* -** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are -** checkpointed. +** Public interfaces: +** +** sqlite3ConnectionBlocked() +** sqlite3ConnectionUnlocked() +** sqlite3ConnectionClosed() +** sqlite3_unlock_notify() */ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ -#ifdef SQLITE_OMIT_WAL - return SQLITE_OK; -#else - int rc; /* Return code */ - int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ - sqlite3_mutex_enter(db->mutex); - if( zDb && zDb[0] ){ - iDb = sqlite3FindDbName(db, zDb); - } - if( iDb<0 ){ - rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); - }else{ - rc = sqlite3Checkpoint(db, iDb); - sqlite3Error(db, rc, 0); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -#endif -} +#define assertMutexHeld() \ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) -#ifndef SQLITE_OMIT_WAL /* -** Run a checkpoint on database iDb. This is a no-op if database iDb is -** not currently open in WAL mode. +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MASTER +** mutex is held. +*/ +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; + +#ifndef NDEBUG +/* +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: ** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is -** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. ** -** The mutex on database handle db should be held by the caller. The mutex -** associated with the specific b-tree being checkpointed is taken by -** this function while the checkpoint is running. +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. ** -** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are -** checkpointed. If an error is encountered it is returned immediately - -** no attempt is made to checkpoint any remaining databases. +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. */ -SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterate through attached dbs */ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; - assert( sqlite3_mutex_held(db->mutex) ); + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); - for(i=0; inDb && rc==SQLITE_OK; i++){ - if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt); + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); } } +} +#else +# define checkListProperties(x) +#endif - return rc; +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; + } + } } -#endif /* SQLITE_OMIT_WAL */ /* -** This function returns true if main-memory should be used instead of -** a temporary file for transient pager files and statement journals. -** The value returned depends on the value of db->temp_store (runtime -** parameter) and the compile time value of SQLITE_TEMP_STORE. The -** following table describes the relationship between these two values -** and this functions return value. -** -** SQLITE_TEMP_STORE db->temp_store Location of temporary database -** ----------------- -------------- ------------------------------ -** 0 any file (return 0) -** 1 1 file (return 0) -** 1 2 memory (return 1) -** 1 0 file (return 0) -** 2 1 file (return 0) -** 2 2 memory (return 1) -** 2 0 memory (return 1) -** 3 any memory (return 1) +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. */ -SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ -#if SQLITE_TEMP_STORE==1 - return ( db->temp_store==2 ); -#endif -#if SQLITE_TEMP_STORE==2 - return ( db->temp_store!=1 ); -#endif -#if SQLITE_TEMP_STORE==3 - return 1; -#endif -#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 - return 0; -#endif +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; } /* -** Return UTF-8 encoded English language explanation of the most recent -** error. +** Obtain the STATIC_MASTER mutex. */ -SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ - const char *z; - if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM); - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return sqlite3ErrStr(SQLITE_MISUSE_BKPT); - } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM); - }else{ - z = (char*)sqlite3_value_text(db->pErr); - assert( !db->mallocFailed ); - if( z==0 ){ - z = sqlite3ErrStr(db->errCode); - } - } - sqlite3_mutex_leave(db->mutex); - return z; +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + checkListProperties(0); } -#ifndef SQLITE_OMIT_UTF16 /* -** Return UTF-16 encoded English language explanation of the most recent -** error. +** Release the STATIC_MASTER mutex. */ -SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ - static const u16 outOfMem[] = { - 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 - }; - static const u16 misuse[] = { - 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', - 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', - 'c', 'a', 'l', 'l', 'e', 'd', ' ', - 'o', 'u', 't', ' ', - 'o', 'f', ' ', - 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 - }; +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; - const void *z; - if( !db ){ - return (void *)outOfMem; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return (void *)misuse; - } sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = (void *)outOfMem; + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); }else{ - z = sqlite3_value_text16(db->pErr); - if( z==0 ){ - sqlite3ValueSetStr(db->pErr, -1, sqlite3ErrStr(db->errCode), - SQLITE_UTF8, SQLITE_STATIC); - z = sqlite3_value_text16(db->pErr); + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); } - /* A malloc() may have failed within the call to sqlite3_value_text16() - ** above. If this is the case, then the db->mallocFailed flag needs to - ** be cleared before returning. Do this directly, instead of via - ** sqlite3ApiExit(), to avoid setting the database handle error message. - */ - db->mallocFailed = 0; } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3Error(db, rc, (rc?"database is deadlocked":0)); sqlite3_mutex_leave(db->mutex); - return z; + return rc; } -#endif /* SQLITE_OMIT_UTF16 */ /* -** Return the most recent error code generated by an SQLite routine. If NULL is -** passed to this function, we assume a malloc() failed during sqlite3_open(). +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. */ -SQLITE_API int sqlite3_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; - } - return db->errCode & db->errMask; -} -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); } - return db->errCode; + db->pBlockingConnection = pBlocker; + leaveMutex(); } /* -** Create a new collating function for database "db". The name is zName -** and the encoding is enc. +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. */ -static int createCollation( - sqlite3* db, - const char *zName, - u8 enc, - u8 collType, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ - CollSeq *pColl; - int enc2; - int nName = sqlite3Strlen30(zName); - - assert( sqlite3_mutex_held(db->mutex) ); +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - */ - enc2 = enc; - testcase( enc2==SQLITE_UTF16 ); - testcase( enc2==SQLITE_UTF16_ALIGNED ); - if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ - enc2 = SQLITE_UTF16NATIVE; - } - if( enc2SQLITE_UTF16BE ){ - return SQLITE_MISUSE_BKPT; - } + aArg = aStatic; + enterMutex(); /* Enter STATIC_MASTER mutex */ - /* Check if this call is removing or replacing an existing collation - ** sequence. If so, and there are active VMs, return busy. If there - ** are no active VMs, invalidate any pre-compiled statements. - */ - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); - if( pColl && pColl->xCmp ){ - if( db->activeVdbeCnt ){ - sqlite3Error(db, SQLITE_BUSY, - "unable to delete/modify collation sequence due to active statements"); - return SQLITE_BUSY; + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; + + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; } - sqlite3ExpirePreparedStatements(db); - /* If collation sequence pColl was created directly by a call to - ** sqlite3_create_collation, and not generated by synthCollSeq(), - ** then any copies made by synthCollSeq() need to be invalidated. - ** Also, collation destructor - CollSeq.xDel() - function may need - ** to be called. - */ - if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); - int j; - for(j=0; j<3; j++){ - CollSeq *p = &aColl[j]; - if( p->enc==pColl->enc ){ - if( p->xDel ){ - p->xDel(p->pUser); - } - p->xCmp = 0; + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; } } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; } } - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM; - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - pColl->type = collType; - sqlite3Error(db, SQLITE_OK, 0); - return SQLITE_OK; + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MASTER mutex */ } - -/* -** This array defines hard upper bounds on limit values. The -** initializer must be kept in sync with the SQLITE_LIMIT_* -** #defines in sqlite3.h. -*/ -static const int aHardLimit[] = { - SQLITE_MAX_LENGTH, - SQLITE_MAX_SQL_LENGTH, - SQLITE_MAX_COLUMN, - SQLITE_MAX_EXPR_DEPTH, - SQLITE_MAX_COMPOUND_SELECT, - SQLITE_MAX_VDBE_OP, - SQLITE_MAX_FUNCTION_ARG, - SQLITE_MAX_ATTACHED, - SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, - SQLITE_MAX_TRIGGER_DEPTH, -}; - /* -** Make sure the hard limits are set to reasonable values +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. */ -#if SQLITE_MAX_LENGTH<100 -# error SQLITE_MAX_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH<100 -# error SQLITE_MAX_SQL_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH -# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH -#endif -#if SQLITE_MAX_COMPOUND_SELECT<2 -# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 -#endif -#if SQLITE_MAX_VDBE_OP<40 -# error SQLITE_MAX_VDBE_OP must be at least 40 -#endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 -#endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>30 -# error SQLITE_MAX_ATTACHED must be between 0 and 30 -#endif -#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 -# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 -#endif -#if SQLITE_MAX_COLUMN>32767 -# error SQLITE_MAX_COLUMN must not exceed 32767 -#endif -#if SQLITE_MAX_TRIGGER_DEPTH<1 -# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} #endif - +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ /* -** Change the value of a limit. Report the old value. -** If an invalid limit index is supplied, report -1. -** Make no changes but still report the old value if the -** new limit is negative. +** 2006 Oct 10 ** -** A new lower limit does not shrink existing constructs. -** It merely prevents new constructs that exceed the limit -** from forming. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. */ -SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ - int oldLimit; - - - /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME - ** there is a hard upper bound set at compile-time by a C preprocessor - ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to - ** "_MAX_".) - */ - assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); - assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); - assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); - assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); - assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); - assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== - SQLITE_MAX_LIKE_PATTERN_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); - assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); - - - if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ - return -1; - } - oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ /* IMP: R-52476-28732 */ - if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ - } - db->aLimit[limitId] = newLimit; - } - return oldLimit; /* IMP: R-53341-35419 */ -} /* -** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" -** is UTF-8 encoded. +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -static int openDatabase( - const char *zFilename, /* Database filename UTF-8 encoded */ - sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned flags, /* Operational flags */ - const char *zVfs /* Name of the VFS to use */ -){ - sqlite3 *db; - int rc; - int isThreadsafe; - - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - - /* Only allow sensible combinations of bits in the flags argument. - ** Throw an error if any non-sense combination is used. If we - ** do not block illegal combinations here, it could trigger - ** assert() statements in deeper layers. Sensible combinations - ** are: - ** - ** 1: SQLITE_OPEN_READONLY - ** 2: SQLITE_OPEN_READWRITE - ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE - */ - assert( SQLITE_OPEN_READONLY == 0x01 ); - assert( SQLITE_OPEN_READWRITE == 0x02 ); - assert( SQLITE_OPEN_CREATE == 0x04 ); - testcase( (1<<(flags&7))==0x02 ); /* READONLY */ - testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ - testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE; - - if( sqlite3GlobalConfig.bCoreMutex==0 ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_NOMUTEX ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_FULLMUTEX ){ - isThreadsafe = 1; - }else{ - isThreadsafe = sqlite3GlobalConfig.bFullMutex; - } - if( flags & SQLITE_OPEN_PRIVATECACHE ){ - flags &= ~SQLITE_OPEN_SHAREDCACHE; - }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ - flags |= SQLITE_OPEN_SHAREDCACHE; - } - /* Remove harmful bits from the flags parameter - ** - ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were - ** dealt with in the previous code block. Besides these, the only - ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, - ** SQLITE_OPEN_READWRITE, and SQLITE_OPEN_CREATE. Silently mask - ** off all other flags. - */ - flags &= ~( SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | - SQLITE_OPEN_MASTER_JOURNAL | - SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX | - SQLITE_OPEN_WAL - ); +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the next docid. It has one position 72 (72-2) and then +** terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +** +** TODO(shess) Provide a VACUUM type operation to clear out all +** deletions and duplications. This would basically be a forced merge +** into a single segment. +*/ - /* Allocate the sqlite data structure */ - db = sqlite3MallocZero( sizeof(sqlite3) ); - if( db==0 ) goto opendb_out; - if( isThreadsafe ){ - db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( db->mutex==0 ){ - sqlite3_free(db); - db = 0; - goto opendb_out; - } - } - sqlite3_mutex_enter(db->mutex); - db->errMask = 0xff; - db->nDb = 2; - db->magic = SQLITE_MAGIC_BUSY; - db->aDb = db->aDbStatic; +/************** Include fts3Int.h in the middle of fts3.c ********************/ +/************** Begin file fts3Int.h *****************************************/ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H - assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); - memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->autoCommit = 1; - db->nextAutovac = -1; - db->nextPagesize = 0; - db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex -#if SQLITE_DEFAULT_FILE_FORMAT<4 - | SQLITE_LegacyFileFmt -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - | SQLITE_LoadExtension -#endif -#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS - | SQLITE_RecTriggers +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif - ; - sqlite3HashInit(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3HashInit(&db->aModule); + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 #endif - db->pVfs = sqlite3_vfs_find(zVfs); - if( !db->pVfs ){ - rc = SQLITE_ERROR; - sqlite3Error(db, rc, "no such vfs: %s", zVfs); - goto opendb_out; - } +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 - ** and UTF-16, so add a version for each to avoid any unnecessary - ** conversions. The only error that can occur here is a malloc() failure. - */ - createCollation(db, "BINARY", SQLITE_UTF8, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, SQLITE_COLL_BINARY, 0, - binCollFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, SQLITE_COLL_USER, (void*)1, - binCollFunc, 0); - if( db->mallocFailed ){ - goto opendb_out; - } - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); - assert( db->pDfltColl!=0 ); +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ - /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, - nocaseCollatingFunc, 0); +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; - /* Open the backend database driver */ - db->openFlags = flags; - rc = sqlite3BtreeOpen(zFilename, db, &db->aDb[0].pBt, 0, - flags | SQLITE_OPEN_MAIN_DB); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; - } - sqlite3Error(db, rc, 0); - goto opendb_out; - } - db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); - db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); +struct sqlite3_tokenizer_module { + /* + ** Structure version. Should always be set to 0. + */ + int iVersion; - /* The default safety_level for the main database is 'full'; for the temp - ** database it is 'NONE'. This matches the pager layer defaults. + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialised by + ** this callback. The caller will do so. */ - db->aDb[0].zName = "main"; - db->aDb[0].safety_level = 3; - db->aDb[1].zName = "temp"; - db->aDb[1].safety_level = 1; + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); - db->magic = SQLITE_MAGIC_OPEN; - if( db->mallocFailed ){ - goto opendb_out; - } + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); - /* Register all built-in functions, but do not attempt to read the - ** database schema yet. This is delayed until the first time the database - ** is accessed. + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). */ - sqlite3Error(db, SQLITE_OK, 0); - sqlite3RegisterBuiltinFunctions(db); + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); - /* Load automatic extensions - extensions that have been registered - ** using the sqlite3_automatic_extension() API. + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). */ - sqlite3AutoLoadExtensions(db); - rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; - } + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); -#ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed ){ - extern int sqlite3Fts1Init(sqlite3*); - rc = sqlite3Fts1Init(db); - } -#endif + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); +}; -#ifdef SQLITE_ENABLE_FTS2 - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlite3Fts2Init(sqlite3*); - rc = sqlite3Fts2Init(db); - } -#endif +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; -#ifdef SQLITE_ENABLE_FTS3 - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts3Init(db); - } -#endif +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; -#ifdef SQLITE_ENABLE_ICU - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3IcuInit(db); - } -#endif +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); -#ifdef SQLITE_ENABLE_RTREE - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3RtreeInit(db); - } -#endif - sqlite3Error(db, rc, 0); +#endif /* _FTS3_TOKENIZER_H_ */ - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking - ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking - ** mode. Doing nothing at all also makes NORMAL the default. - */ -#ifdef SQLITE_DEFAULT_LOCKING_MODE - db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); -#endif +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implemenation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; -opendb_out: - if( db ){ - assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); - sqlite3_mutex_leave(db->mutex); - } - rc = sqlite3_errcode(db); - if( rc==SQLITE_NOMEM ){ - sqlite3_close(db); - db = 0; - }else if( rc!=SQLITE_OK ){ - db->magic = SQLITE_MAGIC_SICK; - } - *ppDb = db; - return sqlite3ApiExit(0, rc); -} +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; /* -** Open a new database handle. +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. */ -SQLITE_API int sqlite3_open( - const char *zFilename, - sqlite3 **ppDb -){ - return openDatabase(zFilename, ppDb, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); -} -SQLITE_API int sqlite3_open_v2( - const char *filename, /* Database filename (UTF-8) */ - sqlite3 **ppDb, /* OUT: SQLite db handle */ - int flags, /* Flags */ - const char *zVfs /* Name of VFS module to use */ -){ - return openDatabase(filename, ppDb, flags, zVfs); -} +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 -#ifndef SQLITE_OMIT_UTF16 /* -** Open a new database handle. +** Access routines. To delete, insert a NULL pointer. */ -SQLITE_API int sqlite3_open16( - const void *zFilename, - sqlite3 **ppDb -){ - char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ - sqlite3_value *pVal; - int rc; - - assert( zFilename ); - assert( ppDb ); - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zFilename8 ){ - rc = openDatabase(zFilename8, ppDb, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); - assert( *ppDb || rc==SQLITE_NOMEM ); - if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ - ENC(*ppDb) = SQLITE_UTF16NATIVE; - } - }else{ - rc = SQLITE_NOMEM; - } - sqlite3ValueFree(pVal); - - return sqlite3ApiExit(0, rc); -} -#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); /* -** Register a new collation sequence with the database handle db. +** Shorthand for the functions above */ -SQLITE_API int sqlite3_create_collation( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem /* -** Register a new collation sequence with the database handle db. +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } */ -SQLITE_API int sqlite3_create_collation_v2( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, xDel); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) -#ifndef SQLITE_OMIT_UTF16 /* -** Register a new collation sequence with the database handle db. +** Number of entries in a hash table */ -SQLITE_API int sqlite3_create_collation16( - sqlite3* db, - const void *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - int rc = SQLITE_OK; - char *zName8; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); - if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); - sqlite3DbFree(db, zName8); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif /* SQLITE_OMIT_UTF16 */ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ + +/************** End of fts3_hash.h *******************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. */ -SQLITE_API int sqlite3_collation_needed( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) -){ - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = xCollNeeded; - db->xCollNeeded16 = 0; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} +#define FTS3_MERGE_COUNT 16 -#ifndef SQLITE_OMIT_UTF16 /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. */ -SQLITE_API int sqlite3_collation_needed16( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) -){ - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = 0; - db->xCollNeeded16 = xCollNeeded16; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_UTF16 */ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) -#ifndef SQLITE_OMIT_DEPRECATED /* -** This function is now an anachronism. It used to be used to recover from a -** malloc() failure, but SQLite now does this automatically. +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. */ -SQLITE_API int sqlite3_global_recover(void){ - return SQLITE_OK; -} +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) #endif /* -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. -** -******* THIS IS AN EXPERIMENTAL API AND IS SUBJECT TO CHANGE ****** +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. */ -SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ - return db->autoCommit; -} +#define FTS3_VARINT_MAX 10 /* -** The following routines are subtitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They server two purposes: +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. ** -** 1. Serve as a convenient place to set a breakpoint in a debugger -** to detect when version error conditions occurs. +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. ** -** 2. Invoke sqlite3_log() to provide the source code location where -** a low-level error is first detected. +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ -SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CORRUPT, - "database corruption at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CORRUPT; -} -SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_MISUSE, - "misuse at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_MISUSE; -} -SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CANTOPEN, - "cannot open file at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CANTOPEN; -} - +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" -#ifndef SQLITE_OMIT_DEPRECATED /* -** This is a convenience routine that makes sure that all thread-specific -** data for this thread has been deallocated. -** -** SQLite no longer uses thread-specific data so this routine is now a -** no-op. It is retained for historical compatibility. +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. */ -SQLITE_API void sqlite3_thread_cleanup(void){ -} +#ifndef testcase +# define testcase(X) #endif /* -** Return meta information about a specific column of a database table. -** See comment in sqlite3.h (sqlite.h.in) for details. +** Terminator values for position-lists and column-lists. */ -#ifdef SQLITE_ENABLE_COLUMN_METADATA -SQLITE_API int sqlite3_table_column_metadata( - sqlite3 *db, /* Connection handle */ - const char *zDbName, /* Database name or NULL */ - const char *zTableName, /* Table name */ - const char *zColumnName, /* Column name */ - char const **pzDataType, /* OUTPUT: Declared data type */ - char const **pzCollSeq, /* OUTPUT: Collation sequence name */ - int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ - int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if column is auto-increment */ -){ - int rc; - char *zErrMsg = 0; - Table *pTab = 0; - Column *pCol = 0; - int iCol; - - char const *zDataType = 0; - char const *zCollSeq = 0; - int notnull = 0; - int primarykey = 0; - int autoinc = 0; - - /* Ensure the database schema has been loaded */ - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrMsg); - if( SQLITE_OK!=rc ){ - goto error_out; - } - - /* Locate the table in question */ - pTab = sqlite3FindTable(db, zTableName, zDbName); - if( !pTab || pTab->pSelect ){ - pTab = 0; - goto error_out; - } - - /* Find the column for which info is requested */ - if( sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - if( iCol>=0 ){ - pCol = &pTab->aCol[iCol]; - } - }else{ - for(iCol=0; iColnCol; iCol++){ - pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ - pTab = 0; - goto error_out; - } - } - - /* The following block stores the meta information that will be returned - ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey - ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. - ** - ** 2. The table is not a view and the column name identified an - ** explicitly declared column. Copy meta information from *pCol. - */ - if( pCol ){ - zDataType = pCol->zType; - zCollSeq = pCol->zColl; - notnull = pCol->notNull!=0; - primarykey = pCol->isPrimKey!=0; - autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; - }else{ - zDataType = "INTEGER"; - primarykey = 1; - } - if( !zCollSeq ){ - zCollSeq = "BINARY"; - } - -error_out: - sqlite3BtreeLeaveAll(db); - - /* Whether the function call succeeded or failed, set the output parameters - ** to whatever their local counterparts contain. If an error did occur, - ** this has the effect of zeroing all output parameters. - */ - if( pzDataType ) *pzDataType = zDataType; - if( pzCollSeq ) *pzCollSeq = zCollSeq; - if( pNotNull ) *pNotNull = notnull; - if( pPrimaryKey ) *pPrimaryKey = primarykey; - if( pAutoinc ) *pAutoinc = autoinc; +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ - if( SQLITE_OK==rc && !pTab ){ - sqlite3DbFree(db, zErrMsg); - zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, - zColumnName); - rc = SQLITE_ERROR; - } - sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); - sqlite3DbFree(db, zErrMsg); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#else +# define ALWAYS(x) (x) +# define NEVER(X) (x) #endif /* -** Sleep for a little while. Return the amount of time slept. +** Internal types used by SQLite. */ -SQLITE_API int sqlite3_sleep(int ms){ - sqlite3_vfs *pVfs; - int rc; - pVfs = sqlite3_vfs_find(0); - if( pVfs==0 ) return 0; - - /* This function works in milliseconds, but the underlying OsSleep() - ** API uses microseconds. Hence the 1000's. - */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); - return rc; -} +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ /* -** Enable or disable the extended result codes. +** Macro used to suppress compiler warnings for unused parameters. */ -SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ - sqlite3_mutex_enter(db->mutex); - db->errMask = onoff ? 0xffffffff : 0xff; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} +#define UNUSED_PARAMETER(x) (void)(x) /* -** Invoke the xFileControl method on a particular database. +** Activate assert() only if SQLITE_TEST is enabled. */ -SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ - int rc = SQLITE_ERROR; - int iDb; - sqlite3_mutex_enter(db->mutex); - if( zDbName==0 ){ - iDb = 0; - }else{ - for(iDb=0; iDbnDb; iDb++){ - if( strcmp(db->aDb[iDb].zName, zDbName)==0 ) break; - } - } - if( iDbnDb ){ - Btree *pBtree = db->aDb[iDb].pBt; - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); - } - sqlite3BtreeLeave(pBtree); - } - } - sqlite3_mutex_leave(db->mutex); - return rc; -} +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif /* -** Interface to the testing logic. +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. */ -SQLITE_API int sqlite3_test_control(int op, ...){ - int rc = 0; -#ifndef SQLITE_OMIT_BUILTIN_TEST - va_list ap; - va_start(ap, op); - switch( op ){ - - /* - ** Save the current state of the PRNG. - */ - case SQLITE_TESTCTRL_PRNG_SAVE: { - sqlite3PrngSaveState(); - break; - } - - /* - ** Restore the state of the PRNG to the last state saved using - ** PRNG_SAVE. If PRNG_SAVE has never before been called, then - ** this verb acts like PRNG_RESET. - */ - case SQLITE_TESTCTRL_PRNG_RESTORE: { - sqlite3PrngRestoreState(); - break; - } - - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. - */ - case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3PrngResetState(); - break; - } +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif - /* - ** sqlite3_test_control(BITVEC_TEST, size, program) - ** - ** Run a test against a Bitvec object of size. The program argument - ** is an array of integers that defines the test. Return -1 on a - ** memory allocation error, 0 on success, or non-zero for an error. - ** See the sqlite3BitvecBuiltinTest() for additional information. - */ - case SQLITE_TESTCTRL_BITVEC_TEST: { - int sz = va_arg(ap, int); - int *aProg = va_arg(ap, int*); - rc = sqlite3BitvecBuiltinTest(sz, aProg); - break; - } +#endif /* SQLITE_AMALGAMATION */ - /* - ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) - ** - ** Register hooks to call to indicate which malloc() failures - ** are benign. - */ - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { - typedef void (*void_function)(void); - void_function xBenignBegin; - void_function xBenignEnd; - xBenignBegin = va_arg(ap, void_function); - xBenignEnd = va_arg(ap, void_function); - sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); - break; - } +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) - ** - ** Set the PENDING byte to the value in the argument, if X>0. - ** Make no changes if X==0. Return the value of the pending byte - ** as it existing before this routine was called. - ** - ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in - ** an incompatible database file format. Changing the PENDING byte - ** while any database connection is open results in undefined and - ** dileterious behavior. - */ - case SQLITE_TESTCTRL_PENDING_BYTE: { - rc = PENDING_BYTE; -#ifndef SQLITE_OMIT_WSD - { - unsigned int newVal = va_arg(ap, unsigned int); - if( newVal ) sqlite3PendingByte = newVal; - } -#endif - break; - } +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) - ** - ** This action provides a run-time test to see whether or not - ** assert() was enabled at compile-time. If X is true and assert() - ** is enabled, then the return value is true. If X is true and - ** assert() is disabled, then the return value is zero. If X is - ** false and assert() is enabled, then the assertion fires and the - ** process aborts. If X is false and assert() is disabled, then the - ** return value is zero. - */ - case SQLITE_TESTCTRL_ASSERT: { - volatile int x = 0; - assert( (x = va_arg(ap,int))!=0 ); - rc = x; - break; - } +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[27]; - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) - ** - ** This action provides a run-time test to see how the ALWAYS and - ** NEVER macros were defined at compile-time. - ** - ** The return value is ALWAYS(X). - ** - ** The recommended test is X==2. If the return value is 2, that means - ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the - ** default setting. If the return value is 1, then ALWAYS() is either - ** hard-coded to true or else it asserts if its argument is false. - ** The first behavior (hard-coded to true) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second - ** behavior (assert if the argument to ALWAYS() is false) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. - ** - ** The run-time test procedure might look something like this: - ** - ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ - ** // ALWAYS() and NEVER() are no-op pass-through macros - ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ - ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. - ** }else{ - ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. - ** } - */ - case SQLITE_TESTCTRL_ALWAYS: { - int x = va_arg(ap,int); - rc = ALWAYS(x); - break; - } + char *zReadExprlist; + char *zWriteExprlist; - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } + int nNodeSize; /* Soft limit for node size */ + u8 bHasStat; /* True if %_stat table exists */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) - ** - ** Enable or disable various optimizations for testing purposes. The - ** argument N is a bitmask of optimizations to be disabled. For normal - ** operation N should be 0. The idea is that a test program (like the - ** SQL Logic Test or SLT test module) can run the same SQL multiple times - ** with various optimizations disabled to verify that the same answer - ** is obtained in every case. - */ - case SQLITE_TESTCTRL_OPTIMIZATIONS: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); - break; - } + /* TODO: Fix the first paragraph of this comment. + ** + ** The following hash table is used to buffer pending index updates during + ** transactions. Variable nPendingData estimates the memory size of the + ** pending data, including hash table overhead, but not malloc overhead. + ** When nPendingData exceeds nMaxPendingData, the buffer is flushed + ** automatically. Variable iPrevDocid is the docid of the most recently + ** inserted record. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ -#ifdef SQLITE_N_KEYWORD - /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) - ** - ** If zWord is a keyword recognized by the parser, then return the - ** number of keywords. Or if zWord is not a keyword, return 0. - ** - ** This test feature is only available in the amalgamation since - ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite - ** is built using separate source files. - */ - case SQLITE_TESTCTRL_ISKEYWORD: { - const char *zWord = va_arg(ap, const char*); - int n = sqlite3Strlen30(zWord); - rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; - break; - } -#endif +#if defined(SQLITE_DEBUG) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribution to the FTS computation; they are used for + ** verifying the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif +}; - /* sqlite3_test_control(SQLITE_TESTCTRL_PGHDRSZ) - ** - ** Return the size of a pcache header in bytes. - */ - case SQLITE_TESTCTRL_PGHDRSZ: { - rc = sizeof(PgHdr); - break; - } +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); - ** - ** Pass pFree into sqlite3ScratchFree(). - ** If sz>0 then allocate a scratch buffer into pNew. - */ - case SQLITE_TESTCTRL_SCRATCHMALLOC: { - void *pFree, **ppNew; - int sz; - sz = va_arg(ap, int); - ppNew = va_arg(ap, void**); - pFree = va_arg(ap, void*); - if( sz ) *ppNew = sqlite3ScratchMalloc(sz); - sqlite3ScratchFree(pFree); - break; - } + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + u32 *aMatchinfo; /* Information about most recent match */ + int nMatchinfo; /* Number of elements in aMatchinfo[] */ + char *zMatchinfo; /* Matchinfo specification */ +}; - } - va_end(ap); -#endif /* SQLITE_OMIT_BUILTIN_TEST */ - return rc; -} +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 -/************** End of main.c ************************************************/ -/************** Begin file notify.c ******************************************/ /* -** 2009 March 3 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in ** -** This file contains the implementation of the sqlite3_unlock_notify() -** API method and its associated functionality. +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. */ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ -/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +} doclist; /* -** Public interfaces: -** -** sqlite3ConnectionBlocked() -** sqlite3ConnectionUnlocked() -** sqlite3ConnectionClosed() -** sqlite3_unlock_notify() +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. */ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ -#define assertMutexHeld() \ - assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; /* -** Head of a linked list of all sqlite3 objects created by this process -** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection -** is not NULL. This variable may only accessed while the STATIC_MASTER -** mutex is held. +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ -static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + u32 *aMI; +}; -#ifndef NDEBUG /* -** This function is a complex assert() that verifies the following -** properties of the blocked connections list: +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: ** -** 1) Each entry in the list has a non-NULL value for either -** pUnlockConnection or pBlockingConnection, or both. +** "a OR b AND c NOT d NEAR e" ** -** 2) All entries in the list that share a common value for -** xUnlockNotify are grouped together. +** is equivalent to: ** -** 3) If the argument db is not NULL, then none of the entries in the -** blocked connections list have pUnlockConnection or pBlockingConnection -** set to db. This is used when closing connection db. +** "a OR (b AND (c NOT (d NEAR e)))" */ -static void checkListProperties(sqlite3 *db){ - sqlite3 *p; - for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ - int seen = 0; - sqlite3 *p2; +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 - /* Verify property (1) */ - assert( p->pUnlockConnection || p->pBlockingConnection ); - /* Verify property (2) */ - for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ - if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; - assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); - assert( db==0 || p->pUnlockConnection!=db ); - assert( db==0 || p->pBlockingConnection!=db ); - } - } -} -#else -# define checkListProperties(x) +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +/* fts3.c */ +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); + +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); + +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); + +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, + char **, int, int, const char *, int, Fts3Expr ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); #endif -/* -** Remove connection db from the blocked connections list. If connection -** db is not currently a part of the list, this function is a no-op. +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + +SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +); + +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); +SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + +#endif /* SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ + +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + + +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT1 +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. */ -static void removeFromBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ - if( *pp==db ){ - *pp = (*pp)->pNextBlocked; - break; - } - } +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); } -/* -** Add connection db to the blocked connections list. It is assumed -** that it is not already a part of the list. +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. */ -static void addToBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for( - pp=&sqlite3BlockedList; - *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; - pp=&(*pp)->pNextBlocked - ); - db->pNextBlocked = *pp; - *pp = db; +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ + const unsigned char *q = (const unsigned char *) p; + sqlite_uint64 x = 0, y = 1; + while( (*q&0x80)==0x80 && q-(unsigned char *)p>= 7; + }while( v!=0 ); + return i; } /* -** Register an unlock-notify callback. -** -** This is called after connection "db" has attempted some operation -** but has received an SQLITE_LOCKED error because another connection -** (call it pOther) in the same process was busy using the same shared -** cache. pOther is found by looking at db->pBlockingConnection. -** -** If there is no blocking connection, the callback is invoked immediately, -** before this routine returns. +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. ** -** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate -** a deadlock. +** Examples: ** -** Otherwise, make arrangements to invoke xNotify when pOther drops -** its locks. +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno ** -** Each call to this routine overrides any prior callbacks registered -** on the same "db". If xNotify==0 then any prior callbacks are immediately -** cancelled. */ -SQLITE_API int sqlite3_unlock_notify( - sqlite3 *db, - void (*xNotify)(void **, int), - void *pArg -){ - int rc = SQLITE_OK; +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ - sqlite3_mutex_enter(db->mutex); - enterMutex(); + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ - if( xNotify==0 ){ - removeFromBlockedList(db); - db->pBlockingConnection = 0; - db->pUnlockConnection = 0; - db->xUnlockNotify = 0; - db->pUnlockArg = 0; - }else if( 0==db->pBlockingConnection ){ - /* The blocking transaction has been concluded. Or there never was a - ** blocking transaction. In either case, invoke the notify callback - ** immediately. - */ - xNotify(&pArg, 1); - }else{ - sqlite3 *p; + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; - for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} - if( p ){ - rc = SQLITE_LOCKED; /* Deadlock detected. */ - }else{ - db->pUnlockConnection = db->pBlockingConnection; - db->xUnlockNotify = xNotify; - db->pUnlockArg = pArg; - removeFromBlockedList(db); - addToBlockedList(db); + while( ALWAYS(z[iIn]) ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } } + z[iOut] = '\0'; } +} - leaveMutex(); - assert( !db->mallocFailed ); - sqlite3Error(db, rc, (rc?"database is deadlocked":0)); - sqlite3_mutex_leave(db->mutex); - return rc; +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; } /* -** This function is called while stepping or preparing a statement -** associated with connection db. The operation will return SQLITE_LOCKED -** to the user because it requires a lock that will not be available -** until connection pBlocker concludes its current transaction. +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. */ -SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ - enterMutex(); - if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ - addToBlockedList(db); +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p = *pp; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); } - db->pBlockingConnection = pBlocker; - leaveMutex(); + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; } /* -** This function is called when -** the transaction opened by database db has just finished. Locks held -** by database connection db have been released. -** -** This function loops through each entry in the blocked connections -** list and does the following: -** -** 1) If the sqlite3.pBlockingConnection member of a list entry is -** set to db, then set pBlockingConnection=0. -** -** 2) If the sqlite3.pUnlockConnection member of a list entry is -** set to db, then invoke the configured unlock-notify callback and -** set pUnlockConnection=0. +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. ** -** 3) If the two steps above mean that pBlockingConnection==0 and -** pUnlockConnection==0, remove the entry from the blocked connections -** list. +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. */ -SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ - void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ - int nArg = 0; /* Number of entries in aArg[] */ - sqlite3 **pp; /* Iterator variable */ - void **aArg; /* Arguments to the unlock callback */ - void **aDyn = 0; /* Dynamically allocated space for aArg[] */ - void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + int rc = SQLITE_OK; /* Return code */ + Fts3Table *p = (Fts3Table *)pVtab; + sqlite3 *db = p->db; - aArg = aStatic; - enterMutex(); /* Enter STATIC_MASTER mutex */ + /* Drop the shadow tables */ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); - /* This loop runs once for each entry in the blocked-connections list. */ - for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ - sqlite3 *p = *pp; + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} - /* Step 1. */ - if( p->pBlockingConnection==db ){ - p->pBlockingConnection = 0; - } - /* Step 2. */ - if( p->pUnlockConnection==db ){ - assert( p->xUnlockNotify ); - if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ - xUnlockNotify(aArg, nArg); - nArg = 0; - } +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ - sqlite3BeginBenignMalloc(); - assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); - assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); - if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) - ){ - /* The aArg[] array needs to grow. */ - void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); - if( pNew ){ - memcpy(pNew, aArg, nArg*sizeof(void *)); - sqlite3_free(aDyn); - aDyn = aArg = pNew; - }else{ - /* This occurs when the array of context pointers that need to - ** be passed to the unlock-notify callback is larger than the - ** aStatic[] array allocated on the stack and the attempt to - ** allocate a larger array from the heap has failed. - ** - ** This is a difficult situation to handle. Returning an error - ** code to the caller is insufficient, as even if an error code - ** is returned the transaction on connection db will still be - ** closed and the unlock-notify callbacks on blocked connections - ** will go unissued. This might cause the application to wait - ** indefinitely for an unlock-notify callback that will never - ** arrive. - ** - ** Instead, invoke the unlock-notify callback with the context - ** array already accumulated. We can then clear the array and - ** begin accumulating any further context pointers without - ** requiring any dynamic allocation. This is sub-optimal because - ** it means that instead of one callback with a large array of - ** context pointers the application will receive two or more - ** callbacks with smaller arrays of context pointers, which will - ** reduce the applications ability to prioritize multiple - ** connections. But it is the best that can be done under the - ** circumstances. - */ - xUnlockNotify(aArg, nArg); - nArg = 0; - } - } - sqlite3EndBenignMalloc(); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); - aArg[nArg++] = p->pUnlockArg; - xUnlockNotify = p->xUnlockNotify; - p->pUnlockConnection = 0; - p->xUnlockNotify = 0; - p->pUnlockArg = 0; + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); } - /* Step 3. */ - if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ - /* Remove connection p from the blocked connections list. */ - *pp = p->pNextBlocked; - p->pNextBlocked = 0; + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; }else{ - pp = &p->pNextBlocked; + rc = sqlite3_declare_vtab(p->db, zSql); } - } - if( nArg!=0 ){ - xUnlockNotify(aArg, nArg); + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; } - sqlite3_free(aDyn); - leaveMutex(); /* Leave STATIC_MASTER mutex */ -} - -/* -** This is called when the database connection passed as an argument is -** being closed. The connection is removed from the blocked list. -*/ -SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ - sqlite3ConnectionUnlocked(db); - enterMutex(); - removeFromBlockedList(db); - checkListProperties(db); - leaveMutex(); } -#endif -/************** End of notify.c **********************************************/ -/************** Begin file fts3.c ********************************************/ /* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. ** -** This is an SQLite module implementing full-text search. +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. */ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + char *zContentCols; /* Columns of %_content table */ + sqlite3 *db = p->db; /* The database connection */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + } + return rc; +} /* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or +** Store the current database page-size in bytes in p->nPgsz. ** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ - -/* The full-text index is stored in a series of b+tree (-like) -** structures called segments which map terms to doclists. The -** structures are like b+trees in layout, but are constructed from the -** bottom up in optimal fashion and are not updatable. Since trees -** are built from the bottom up, things will be described from the -** bottom up. -** -** -**** Varints **** -** The basic unit of encoding is a variable-length integer called a -** varint. We encode variable-length integers in little-endian order -** using seven bits * per byte as follows: -** -** KEY: -** A = 0xxxxxxx 7 bits of data and one flag bit -** B = 1xxxxxxx 7 bits of data and one flag bit -** -** 7 bits - A -** 14 bits - BA -** 21 bits - BBA -** and so on. -** -** This is similar in concept to how sqlite encodes "varints" but -** the encoding is not the same. SQLite varints are big-endian -** are are limited to 9 bytes in length whereas FTS3 varints are -** little-endian and can be up to 10 bytes in length (in theory). -** -** Example encodings: -** -** 1: 0x01 -** 127: 0x7f -** 128: 0x81 0x00 -** -** -**** Document lists **** -** A doclist (document list) holds a docid-sorted list of hits for a -** given term. Doclists hold docids and associated token positions. -** A docid is the unique integer identifier for a single document. -** A position is the index of a word within the document. The first -** word of the document has a position of 0. -** -** FTS3 used to optionally store character offsets using a compile-time -** option. But that functionality is no longer supported. -** -** A doclist is stored like this: -** -** array { -** varint docid; -** array { (position list for column 0) -** varint position; (2 more than the delta from previous position) -** } -** array { -** varint POS_COLUMN; (marks start of position list for new column) -** varint column; (index of new column) -** array { -** varint position; (2 more than the delta from previous position) -** } -** } -** varint POS_END; (marks end of positions for this document. -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals -** ending a position list array. POS_END is 0. POS_COLUMN is 1. -** The positions numbers are not stored literally but rather as two more -** than the difference from the prior position, or the just the position plus -** 2 for the first position. Example: -** -** label: A B C D E F G H I J K -** value: 123 5 9 1 1 14 35 0 234 72 0 -** -** The 123 value is the first docid. For column zero in this document -** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 -** at D signals the start of a new column; the 1 at E indicates that the -** new column is column number 1. There are two positions at 12 and 45 -** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the next docid. It has one position 72 (72-2) and then -** terminates with the 0 at K. -** -** A "position-list" is the list of positions for multiple columns for -** a single docid. A "column-list" is the set of positions for a single -** column. Hence, a position-list consists of one or more column-lists, -** a document record consists of a docid followed by a position-list and -** a doclist consists of one or more document records. -** -** A bare doclist omits the position information, becoming an -** array of varint-encoded docids. -** -**** Segment leaf nodes **** -** Segment leaf nodes store terms and doclists, ordered by term. Leaf -** nodes are written using LeafWriter, and read using LeafReader (to -** iterate through a single leaf node's data) and LeavesReader (to -** iterate through a segment's entire leaf layer). Leaf nodes have -** the format: -** -** varint iHeight; (height from leaf level, always 0) -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of prefix shared with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix];(unshared suffix of next term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. -** -** Leaf nodes are broken into blocks which are stored contiguously in -** the %_segments table in sorted order. This means that when the end -** of a node is reached, the next term is in the node with the next -** greater node id. -** -** New data is spilled to a new leaf node when the current node -** exceeds LEAF_MAX bytes (default 2048). New data which itself is -** larger than STANDALONE_MIN (default 1024) is placed in a standalone -** node (a leaf node with a single term and doclist). The goal of -** these settings is to pack together groups of small doclists while -** making it efficient to directly access large doclists. The -** assumption is that large doclists represent terms which are more -** likely to be query targets. -** -** TODO(shess) It may be useful for blocking decisions to be more -** dynamic. For instance, it may make more sense to have a 2.5k leaf -** node rather than splitting into 2k and .5k nodes. My intuition is -** that this might extend through 2x or 4x the pagesize. -** -** -**** Segment interior nodes **** -** Segment interior nodes store blockids for subtree nodes and terms -** to describe what data is stored by the each subtree. Interior -** nodes are written using InteriorWriter, and read using -** InteriorReader. InteriorWriters are created as needed when -** SegmentWriter creates new leaf nodes, or when an interior node -** itself grows too big and must be split. The format of interior -** nodes: -** -** varint iHeight; (height from leaf level, always >0) -** varint iBlockid; (block id of node's leftmost subtree) -** optional { -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of shared prefix with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix]; (unshared suffix of next term) -** } -** } -** -** Here, optional { X } means an optional element, while array { X } -** means zero or more occurrences of X, adjacent in memory. -** -** An interior node encodes n terms separating n+1 subtrees. The -** subtree blocks are contiguous, so only the first subtree's blockid -** is encoded. The subtree at iBlockid will contain all terms less -** than the first term encoded (or all terms if no term is encoded). -** Otherwise, for terms greater than or equal to pTerm[i] but less -** than pTerm[i+1], the subtree for that term will be rooted at -** iBlockid+i. Interior nodes only store enough term data to -** distinguish adjacent children (if the rightmost term of the left -** child is "something", and the leftmost term of the right child is -** "wicked", only "w" is stored). -** -** New data is spilled to a new interior node at the same height when -** the current node exceeds INTERIOR_MAX bytes (default 2048). -** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing -** interior nodes and making the tree too skinny. The interior nodes -** at a given height are naturally tracked by interior nodes at -** height+1, and so on. -** -** -**** Segment directory **** -** The segment directory in table %_segdir stores meta-information for -** merging and deleting segments, and also the root node of the -** segment's tree. -** -** The root node is the top node of the segment's tree after encoding -** the entire segment, restricted to ROOT_MAX bytes (default 1024). -** This could be either a leaf node or an interior node. If the top -** node requires more than ROOT_MAX bytes, it is flushed to %_segments -** and a new root interior node is generated (which should always fit -** within ROOT_MAX because it only needs space for 2 varints, the -** height and the blockid of the previous root). -** -** The meta-information in the segment directory is: -** level - segment level (see below) -** idx - index within level -** - (level,idx uniquely identify a segment) -** start_block - first leaf node -** leaves_end_block - last leaf node -** end_block - last block (including interior nodes) -** root - contents of root node -** -** If the root node is a leaf node, then start_block, -** leaves_end_block, and end_block are all 0. -** -** -**** Segment merging **** -** To amortize update costs, segments are grouped into levels and -** merged in batches. Each increase in level represents exponentially -** more documents. -** -** New documents (actually, document updates) are tokenized and -** written individually (using LeafWriter) to a level 0 segment, with -** incrementing idx. When idx reaches MERGE_COUNT (default 16), all -** level 0 segments are merged into a single level 1 segment. Level 1 -** is populated like level 0, and eventually MERGE_COUNT level 1 -** segments are merged to a single level 2 segment (representing -** MERGE_COUNT^2 updates), and so on. -** -** A segment merge traverses all segments at a given level in -** parallel, performing a straightforward sorted merge. Since segment -** leaf nodes are written in to the %_segments table in order, this -** merge traverses the underlying sqlite disk structures efficiently. -** After the merge, all segment blocks from the merged level are -** deleted. -** -** MERGE_COUNT controls how often we merge segments. 16 seems to be -** somewhat of a sweet spot for insertion performance. 32 and 64 show -** very similar performance numbers to 16 on insertion, though they're -** a tiny bit slower (perhaps due to more overhead in merge-time -** sorting). 8 is about 20% slower than 16, 4 about 50% slower than -** 16, 2 about 66% slower than 16. -** -** At query time, high MERGE_COUNT increases the number of segments -** which need to be scanned and merged. For instance, with 100k docs -** inserted: -** -** MERGE_COUNT segments -** 16 25 -** 8 12 -** 4 10 -** 2 6 -** -** This appears to have only a moderate impact on queries for very -** frequent terms (which are somewhat dominated by segment merge -** costs), and infrequent and non-existent terms still seem to be fast -** even with many segments. -** -** TODO(shess) That said, it would be nice to have a better query-side -** argument for MERGE_COUNT of 16. Also, it is possible/likely that -** optimizations to things like doclist merging will swing the sweet -** spot around. -** -** -** -**** Handling of deletions and updates **** -** Since we're using a segmented structure, with no docid-oriented -** index into the term index, we clearly cannot simply update the term -** index when a document is deleted or updated. For deletions, we -** write an empty doclist (varint(docid) varint(POS_END)), for updates -** we simply write the new doclist. Segment merges overwrite older -** data for a particular docid with newer data, so deletes or updates -** will eventually overtake the earlier data and knock it out. The -** query logic likewise merges doclists so that newer data knocks out -** older data. +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: ** -** TODO(shess) Provide a VACUUM type operation to clear out all -** deletions and duplications. This would basically be a forced merge -** into a single segment. +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. */ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } -#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) -# define SQLITE_CORE 1 -#endif + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} -/************** Include fts3Int.h in the middle of fts3.c ********************/ -/************** Begin file fts3Int.h *****************************************/ /* -** 2009 Nov 12 +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + va_end(ap); + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + int nRet; + char *zRet; + nRet = 2 + strlen(zInput)*2 + 1; + zRet = sqlite3_malloc(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions that could be used +** in a SELECT statement such as the following: ** -****************************************************************************** +** SELECT FROM %_content AS x ... ** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. */ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; -#ifndef _FTSINT_H -#define _FTSINT_H - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + sqlite3_free(zFree); + return zRet; +} -/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ -/************** Begin file fts3_tokenizer.h **********************************/ /* -** 2006 July 10 -** -** The author disclaims copyright to this source code. +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). ** -************************************************************************* -** Defines the interface to tokenizers used by fulltext-search. There -** are three basic components: +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: ** -** sqlite3_tokenizer_module is a singleton defining the tokenizer -** interface functions. This is essentially the class structure for -** tokenizers. +** "?, zip(?), zip(?), zip(?)" ** -** sqlite3_tokenizer is used to define a particular tokenizer, perhaps -** including customization information defined at creation time. +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. ** -** sqlite3_tokenizer_cursor is generated by a tokenizer to generate -** tokens from a particular input. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. */ -#ifndef _FTS3_TOKENIZER_H_ -#define _FTS3_TOKENIZER_H_ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + sqlite3_free(zFree); + return zRet; +} + +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p = *pp; + int nInt = 0; + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} -/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. -** If tokenizers are to be allowed to call sqlite3_*() functions, then -** we will need a way to register the API consistently. -*/ + +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ +){ + struct Fts3Index *aIndex; + int nIndex = 1; + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = *apFree = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. */ -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; -typedef struct sqlite3_tokenizer sqlite3_tokenizer; -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + int nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ -struct sqlite3_tokenizer_module { + int nIndex; /* Size of aIndex[] array */ + struct Fts3Index *aIndex; /* Array of indexes for this table */ + struct Fts3Index *aFree = 0; /* Free this before returning */ - /* - ** Structure version. Should always be set to 0. - */ - int iVersion; + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - /* - ** Create a new tokenizer. The values in the argv[] array are the - ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts3 table. For example, if - ** the following SQL is executed: - ** - ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) - ** - ** then argc is set to 2, and the argv[] array contains pointers - ** to the strings "arg1" and "arg2". - ** - ** This method should return either SQLITE_OK (0), or an SQLite error - ** code. If SQLITE_OK is returned, then *ppTokenizer should be set - ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialised by - ** this callback. The caller will do so. - */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); - /* - ** Destroy an existing tokenizer. The fts3 module calls this method - ** exactly once for each successful call to xCreate(). - */ - int (*xDestroy)(sqlite3_tokenizer *pTokenizer); - - /* - ** Create a tokenizer cursor to tokenize an input buffer. The caller - ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). - */ - int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ - ); + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; - /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this - ** method exactly once for each successful call to xOpen(). - */ - int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) ); + if( !aCol ) return SQLITE_NOMEM; + memset((void *)aCol, 0, sizeof(const char *) * (argc-2)); - /* - ** Retrieve the next token from the tokenizer cursor pCursor. This - ** method should either return SQLITE_OK and set the values of the - ** "OUT" variables identified below, or SQLITE_DONE to indicate that - ** the end of the buffer has been reached, or an SQLite error code. + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: ** - ** *ppToken should be set to point at a buffer containing the - ** normalized version of the token (i.e. after any case-folding and/or - ** stemming has been performed). *pnBytes should be set to the length - ** of this buffer in bytes. The input text that generated the token is - ** identified by the byte offsets returned in *piStartOffset and - ** *piEndOffset. *piStartOffset should be set to the index of the first - ** byte of the token in the input buffer. *piEndOffset should be set - ** to the index of the first byte just past the end of the token in - ** the input buffer. + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. ** - ** The buffer *ppToken is set to point at is managed by the tokenizer - ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). - */ - /* TODO(shess) current implementation requires pInput to be - ** nul-terminated. This should either be fixed, or pInput/nBytes - ** should be converted to zInput. + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. */ - int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ - ); -}; - -struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ - /* Tokenizer implementations will typically add additional fields */ -}; + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } -int fts3_global_term_cnt(int iTerm, int iCol); -int fts3_term_cnt(int iTerm, int iCol); + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + char **pzVar; + } aFts4Opt[] = { + { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */ + { "prefix", 6, 0 }, /* 1 -> PREFIX */ + { "compress", 8, 0 }, /* 2 -> COMPRESS */ + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ + { "order", 5, 0 } /* 4 -> ORDER */ + }; + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; -#endif /* _FTS3_TOKENIZER_H_ */ + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; -/************** End of fts3_tokenizer.h **************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ -/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ -/************** Begin file fts3_hash.h ***************************************/ -/* -** 2001 September 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the header file for the generic hash-table implemenation -** used in SQLite. We've modified it slightly to serve as a standalone -** hash table implementation for the full-text indexing module. -** -*/ -#ifndef _FTS3_HASH_H_ -#define _FTS3_HASH_H_ + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; -/* Forward declarations of structures. */ -typedef struct Fts3Hash Fts3Hash; -typedef struct Fts3HashElem Fts3HashElem; + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; -/* A complete hash table is an instance of the following structure. -** The internals of this structure are intended to be opaque -- client -** code should not attempt to access or modify the fields of this structure -** directly. Change this structure only by using the routines below. -** However, many of the "procedures" and "functions" for modifying and -** accessing this structure are really macros, so we can't really make -** this structure opaque. -*/ -struct Fts3Hash { - char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ - char copyKey; /* True if copy of key made on insert */ - int count; /* Number of entries in this table */ - Fts3HashElem *first; /* The first element of the array */ - int htsize; /* Number of buckets in the hash table */ - struct _fts3ht { /* the hash table */ - int count; /* Number of entries with this hash */ - Fts3HashElem *chain; /* Pointer to first entry with this hash */ - } *ht; -}; + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + } + } + sqlite3_free(zVal); + } + } -/* Each element in the hash table is an instance of the following -** structure. All elements are stored on a single doubly-linked list. -** -** Again, this structure is intended to be opaque, but it can't really -** be opaque because it is used by macros. -*/ -struct Fts3HashElem { - Fts3HashElem *next, *prev; /* Next and previous elements in the table */ - void *data; /* Data associated with this element */ - void *pKey; int nKey; /* Key associated with this element */ -}; + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + if( rc!=SQLITE_OK ) goto fts3_init_out; -/* -** There are 2 different modes of operation for a hash table: -** -** FTS3_HASH_STRING pKey points to a string that is nKey bytes long -** (including the null-terminator, if any). Case -** is respected in comparisons. -** -** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. -** memcmp() is used to compare keys. -** -** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. -*/ -#define FTS3_HASH_STRING 1 -#define FTS3_HASH_BINARY 2 + if( nCol==0 ){ + assert( nString==0 ); + aCol[0] = "content"; + nString = 8; + nCol = 1; + } -/* -** Access routines. To delete, insert a NULL pointer. -*/ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); -SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + if( pTokenizer==0 ){ + rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); + if( rc!=SQLITE_OK ) goto fts3_init_out; + } + assert( pTokenizer ); -/* -** Shorthand for the functions above -*/ -#define fts3HashInit sqlite3Fts3HashInit -#define fts3HashInsert sqlite3Fts3HashInsert -#define fts3HashFind sqlite3Fts3HashFind -#define fts3HashClear sqlite3Fts3HashClear -#define fts3HashFindElem sqlite3Fts3HashFindElem + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + if( rc==SQLITE_ERROR ){ + assert( zPrefix ); + *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); + } + if( rc!=SQLITE_OK ) goto fts3_init_out; -/* -** Macros for looping over all elements of a hash table. The idiom is -** like this: -** -** Fts3Hash h; -** Fts3HashElem *p; -** ... -** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ -** SomeStructure *pData = fts3HashData(p); -** // do something with pData -** } -*/ -#define fts3HashFirst(H) ((H)->first) -#define fts3HashNext(E) ((E)->next) -#define fts3HashData(E) ((E)->data) -#define fts3HashKey(E) ((E)->pKey) -#define fts3HashKeysize(E) ((E)->nKey) + /* Allocate and populate the Fts3Table structure. */ + nByte = sizeof(Fts3Table) + /* Fts3Table */ + nCol * sizeof(char *) + /* azColumn */ + nIndex * sizeof(struct Fts3Index) + /* aIndex */ + nName + /* zName */ + nDb + /* zDb */ + nString; /* Space for azColumn strings */ + p = (Fts3Table*)sqlite3_malloc(nByte); + if( p==0 ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + memset(p, 0, nByte); + p->db = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; + p->bDescIdx = bDescIdx; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); -/* -** Number of entries in a hash table -*/ -#define fts3HashCount(H) ((H)->count) + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } -#endif /* _FTS3_HASH_H_ */ + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->aIndex[nIndex]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; -/************** End of fts3_hash.h *******************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ + /* Fill in the azColumn array */ + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } -/* -** This constant controls how often segments are merged. Once there are -** FTS3_MERGE_COUNT segments of level N, they are merged into a single -** segment of level N+1. -*/ -#define FTS3_MERGE_COUNT 16 + if( (zCompress==0)!=(zUncompress==0) ){ + char const *zMiss = (zCompress==0 ? "compress" : "uncompress"); + rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("missing %s parameter in fts4 constructor", zMiss); + } + p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; -/* -** This is the maximum amount of data (in bytes) to store in the -** Fts3Table.pendingTerms hash table. Normally, the hash table is -** populated as documents are inserted/updated/deleted in a transaction -** and used to create a new segment when the transaction is committed. -** However if this limit is reached midway through a transaction, a new -** segment is created and the hash table cleared immediately. -*/ -#define FTS3_MAX_PENDING_DATA (1*1024*1024) + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } -/* -** Macro to return the number of elements in an array. SQLite has a -** similar macro called ArraySize(). Use a different name to avoid -** a collision when building an amalgamation with built-in FTS3. -*/ -#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; -/* -** Maximum length of a varint encoded integer. The varint format is different -** from that used by SQLite, so the maximum length is 10, not 9. -*/ -#define FTS3_VARINT_MAX 10 + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); -/* -** The testcase() macro is only used by the amalgamation. If undefined, -** make it a no-op. -*/ -#ifndef testcase -# define testcase(X) -#endif +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aFree); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free((void *)aCol); + if( rc!=SQLITE_OK ){ + if( p ){ + fts3DisconnectMethod((sqlite3_vtab *)p); + }else if( pTokenizer ){ + pTokenizer->pModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} /* -** Terminator values for position-lists and column-lists. +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). */ -#define POS_COLUMN (1) /* Column-list terminator */ -#define POS_END (0) /* Position-list terminator */ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} -/* -** This section provides definitions to allow the -** FTS3 extension to be compiled outside of the -** amalgamation. -*/ -#ifndef SQLITE_AMALGAMATION -/* -** Macros indicating that conditional expressions are always true or -** false. +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. */ -#ifdef SQLITE_COVERAGE_TEST -# define ALWAYS(x) (1) -# define NEVER(X) (0) -#else -# define ALWAYS(x) (x) -# define NEVER(X) (x) -#endif +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ -/* -** Internal types used by SQLite. -*/ -typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ -typedef short int i16; /* 2-byte (or larger) signed integer */ -typedef unsigned int u32; /* 4-byte unsigned integer */ -typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ -/* -** Macro used to suppress compiler warnings for unused parameters. -*/ -#define UNUSED_PARAMETER(x) (void)(x) -#endif + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 500000; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ) continue; -typedef struct Fts3Table Fts3Table; -typedef struct Fts3Cursor Fts3Cursor; -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3PhraseToken Fts3PhraseToken; + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) + ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } -typedef struct Fts3SegFilter Fts3SegFilter; -typedef struct Fts3DeferredToken Fts3DeferredToken; -typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderArray Fts3SegReaderArray; + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + break; + } + } -/* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. -*/ -struct Fts3Table { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of named columns in virtual table */ - char **azColumn; /* column names. malloced */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = 1; + pInfo->aConstraintUsage[iCons].omit = 1; + } - /* Precompiled statements used by the implementation. Each of these - ** statements is run and reset within a single virtual table API call. + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. */ - sqlite3_stmt *aStmt[24]; - - int nNodeSize; /* Soft limit for node size */ - u8 bHasStat; /* True if %_stat table exists */ - u8 bHasDocsize; /* True if %_docsize table exists */ - int nPgsz; /* Page size for host database */ - char *zSegmentsTbl; /* Name of %_segments table */ - sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } - /* The following hash table is used to buffer pending index updates during - ** transactions. Variable nPendingData estimates the memory size of the - ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds nMaxPendingData, the buffer is flushed - ** automatically. Variable iPrevDocid is the docid of the most recently - ** inserted record. - */ - int nMaxPendingData; - int nPendingData; - sqlite_int64 iPrevDocid; - Fts3Hash pendingTerms; -}; + assert( p->pSegments==0 ); + return SQLITE_OK; +} /* -** When the core wants to read from the virtual table, it creates a -** virtual table cursor (an instance of the following structure) using -** the xOpen method. Cursors are destroyed using the xClose method. +** Implementation of xOpen method. */ -struct Fts3Cursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - i16 eSearch; /* Search strategy (see below) */ - u8 isEof; /* True if at End Of Results */ - u8 isRequireSeek; /* True if must seek pStmt to %_content row */ - sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ - Fts3Expr *pExpr; /* Parsed MATCH query string */ - int nPhrase; /* Number of matchable phrases in query */ - Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ - sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ - char *pNextId; /* Pointer into the body of aDoclist */ - char *aDoclist; /* List of docids for full-text queries */ - int nDoclist; /* Size of buffer at aDoclist */ - int eEvalmode; /* An FTS3_EVAL_XX constant */ - int nRowAvg; /* Average size of database rows, in pages */ - - int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - u32 *aMatchinfo; /* Information about most recent match */ - int nMatchinfo; /* Number of elements in aMatchinfo[] */ - char *zMatchinfo; /* Matchinfo specification */ -}; +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ -#define FTS3_EVAL_FILTER 0 -#define FTS3_EVAL_NEXT 1 -#define FTS3_EVAL_MATCHINFO 2 + UNUSED_PARAMETER(pVTab); -/* -** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to -** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index -** of the column to be searched. For example, in -** -** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); -** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; -** -** Because the LHS of the MATCH operator is 2nd column "b", -** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, -** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" -** indicating that all columns should be searched, -** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. -*/ -#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ -#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ -#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} /* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in double-quotes (i.e. "one two three") -** nToken will be the number of tokens in the string. -** -** The nDocMatch and nMatch variables contain data that may be used by the -** matchinfo() function. They are populated when the full-text index is -** queried for hits on the phrase. If one or more tokens in the phrase -** are deferred, the nDocMatch and nMatch variables are populated based -** on the assumption that the +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. */ -struct Fts3PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer z */ - int isPrefix; /* True if token ends with a "*" character */ - int bFulltext; /* True if full-text index was used */ - Fts3SegReaderArray *pArray; /* Segment-reader for this token */ - Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ -}; - -struct Fts3Phrase { - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ -}; +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} /* -** A tree of these objects forms the RHS of a MATCH operator. -** -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, -** containing the results of the NEAR or phrase query in FTS3 doclist -** format. As usual, the initial "Length" field found in doclists stored -** on disk is omitted from this buffer. -** -** Variable pCurrent always points to the start of a docid field within -** aDoclist. Since the doclist is usually scanned in docid order, this can -** be used to accelerate seeking to the required docid within the doclist. +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. */ -struct Fts3Expr { - int eType; /* One of the FTSQUERY_XXX values defined below */ - int nNear; /* Valid if eType==FTSQUERY_NEAR */ - Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ - - sqlite3_int64 iCurrent; - char *pCurrent; -}; +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + if( pCsr->isRequireSeek ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + int rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK ){ + /* If no row was found and no error has occured, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. + */ + rc = SQLITE_CORRUPT_VTAB; + } + pCsr->isEof = 1; + if( pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; + } + }else{ + return SQLITE_OK; + } +} /* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. ** -** "a OR b AND c NOT d NEAR e" +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. ** -** is equivalent to: +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. ** -** "a OR (b AND (c NOT (d NEAR e)))" +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - - -/* fts3_write.c */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, - sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( - Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, - int (*)(Fts3Table *, void *, char *, int, char *, int), void * -); -SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); - -SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); - -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); - -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ -/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ -#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 -#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 -#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 -#define FTS3_SEGMENT_PREFIX 0x00000008 + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return SQLITE_CORRUPT_VTAB; + } + + while( zCsrzEnd ){ + rc = SQLITE_CORRUPT_VTAB; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; -/* Type passed as 4th argument to SegmentReaderIterate() */ -struct Fts3SegFilter { - const char *zTerm; - int nTerm; - int iCol; - int flags; -}; + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } -/* fts3.c */ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } -SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); + iChild++; + }; -/* fts3_tokenizer.c */ -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, - sqlite3_tokenizer **, char ** -); -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; -/* fts3_snippet.c */ -SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); -SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, - const char *, const char *, int, int -); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); + finish_scan: + sqlite3_free(zBuffer); + return rc; +} -/* fts3_expr.c */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, - char **, int, int, const char *, int, Fts3Expr ** -); -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); -#ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); -#endif -#endif /* _FTSINT_H */ +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc; /* Return code */ + int iHeight; /* Height of this node in tree */ -/************** End of fts3Int.h *********************************************/ -/************** Continuing where we left off in fts3.c ***********************/ + assert( piLeaf || piLeaf2 ); + sqlite3Fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob; /* Size of zBlob in bytes */ -/* -** Write a 64-bit variable-length integer to memory starting at p[0]. -** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. -** The number of bytes written is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ - unsigned char *q = (unsigned char *) p; - sqlite_uint64 vu = v; - do{ - *q++ = (unsigned char) ((vu & 0x7f) | 0x80); - vu >>= 7; - }while( vu!=0 ); - q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); - return (int) (q - (unsigned char *)p); -} + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } -/* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. -*/ -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const unsigned char *q = (const unsigned char *) p; - sqlite_uint64 x = 0, y = 1; - while( (*q&0x80)==0x80 && q-(unsigned char *)p 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; } /* -** Return the number of bytes required to encode v as a varint +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. */ -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 ); - return i; +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; } /* -** Convert an SQL-style quoted string into a normal string by removing -** the quote characters. The conversion is done in-place. If the -** input does not begin with a quote character, then this routine -** is a no-op. +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. ** -** Examples: +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. ** -** "abc" becomes abc -** 'xyz' becomes xyz -** [pqr] becomes pqr -** `mno` becomes mno +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. ** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. */ -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ - char quote; /* Quote character (if any ) */ - - quote = z[0]; - if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ - int iIn = 1; /* Index of next byte to read from input */ - int iOut = 0; /* Index of next byte to write to output */ - - /* If the first byte was a '[', then the close-quote character is a ']' */ - if( quote=='[' ) quote = ']'; +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; - while( ALWAYS(z[iIn]) ){ - if( z[iIn]==quote ){ - if( z[iIn+1]!=quote ) break; - z[iOut++] = quote; - iIn += 2; - }else{ - z[iOut++] = z[iIn++]; - } - } - z[iOut] = '\0'; + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; } + *ppPoslist = pEnd; } /* -** Read a single varint from the doclist at *pp and advance *pp to point -** to the first byte past the end of the varint. Add the value of the varint -** to *pVal. +** Value used to signify the end of an position-list. This is safe because +** it is not possible to have a document with 2^31 terms. */ -static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - *pVal += iVal; -} +#define POSITION_LIST_END 0x7fffffff /* -** As long as *pp has not reached its end (pEnd), then do the same -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. -** But if we have reached the end of the varint, just set *pp=0 and -** leave *pVal unchanged. +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. */ -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ - if( *pp>=pEnd ){ - *pp = 0; +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + fts3GetDeltaVarint(pp, pi); + *pi -= 2; }else{ - fts3GetDeltaVarint(pp, pVal); + *pi = POSITION_LIST_END; } } /* -** The xDisconnect() virtual table method. +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). */ -static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int i; +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} - assert( p->nPendingData==0 ); - assert( p->pSegments==0 ); +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(p->aStmt[i]); - } - sqlite3_free(p->zSegmentsTbl); + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ - /* Invoke the tokenizer destructor to free the tokenizer. */ - p->pTokenizer->pModule->xDestroy(p->pTokenizer); + if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); + else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + else iCol1 = 0; - sqlite3_free(p); - return SQLITE_OK; + if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); + else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. */ -static void fts3DbExec( - int *pRc, /* Success code */ - sqlite3 *db, /* Database in which to run SQL */ - const char *zFormat, /* Format string for SQL */ - ... /* Arguments to the format string */ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ ){ - va_list ap; - char *zSql; - if( *pRc ) return; - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - *pRc = sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_free(zSql); + char *p = (pp ? *pp : 0); + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert( *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += sqlite3Fts3GetVarint32(p2, &iCol2); } -} -/* -** The xDestroy() virtual table method. -*/ -static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - int rc = SQLITE_OK; /* Return code */ - Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; - /* Drop the shadow tables */ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); + if( pp && iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } - /* If everything has worked, invoke fts3DisconnectMethod() to free the - ** memory associated with the Fts3Table structure and return SQLITE_OK. - ** Otherwise, return an SQLite error code. - */ - return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); -} + assert( *p1!=POS_END && *p1!=POS_COLUMN ); + assert( *p2!=POS_END && *p2!=POS_COLUMN ); + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + if( !pp ){ + fts3PoslistCopy(0, &p2); + fts3PoslistCopy(0, &p1); + *pp1 = p1; + *pp2 = p2; + return 1; + } + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p2++; + p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName - ); - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; +static void fts3GetDeltaVarint3( + char **pp, + char *pEnd, + int bDescIdx, + sqlite3_int64 *pVal +){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; }else{ - rc = sqlite3_declare_vtab(p->db, zSql); + *pVal += iVal; } + } +} - sqlite3_free(zSql); - sqlite3_free(zCols); - *pRc = rc; +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; } -/* -** Create the backing store tables (%_content, %_segments and %_segdir) -** required by the FTS3 table passed as the only argument. This is done -** as part of the vtab xCreate() method. -** -** If the p->bHasDocsize boolean is true (indicating that this is an -** FTS4 table, not an FTS3 table) then also create the %_docsize and -** %_stat tables required by FTS4. -*/ -static int fts3CreateTables(Fts3Table *p){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - char *zContentCols; /* Columns of %_content table */ - sqlite3 *db = p->db; /* The database connection */ +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); - /* Create other tables */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", - p->zDb, p->zName - ); - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segdir'(" - "level INTEGER," - "idx INTEGER," - "start_block INTEGER," - "leaves_end_block INTEGER," - "end_block INTEGER," - "root BLOB," - "PRIMARY KEY(level, idx)" - ");", - p->zDb, p->zName - ); - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", - p->zDb, p->zName - ); - } - if( p->bHasStat ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } } - return rc; + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; } -/* -** Store the current database page-size in bytes in p->nPgsz. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. -*/ -static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - char *zSql; /* SQL text "PRAGMA %Q.page_size" */ - sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ - - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); - if( !zSql ){ - rc = SQLITE_NOMEM; +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); }else{ - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - p->nPgsz = sqlite3_column_int(pStmt, 0); - rc = sqlite3_finalize(pStmt); - } + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); } - assert( p->nPgsz>0 || rc!=SQLITE_OK ); - sqlite3_free(zSql); - *pRc = rc; } + + *pnRight = p - aOut; } + /* -** "Special" FTS4 arguments are column specifications of the following form: -** -** = +** Merge all doclists in the TermSelect.aaOutput[] array into a single +** doclist stored in TermSelect.aaOutput[0]. If successful, delete all +** other doclists (except the aaOutput[0] one) and return SQLITE_OK. ** -** There may not be whitespace surrounding the "=" character. The -** term may be quoted, but the may not. +** If an OOM error occurs, return SQLITE_NOMEM. In this case it is +** the responsibility of the caller to free any doclists left in the +** TermSelect.aaOutput[] array. */ -static int fts3IsSpecialColumn( - const char *z, - int *pnKey, - char **pzValue -){ - char *zValue; - const char *zCsr = z; +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ + char *aOut = 0; + int nOut = 0; + int i; - while( *zCsr!='=' ){ - if( *zCsr=='\0' ) return 0; - zCsr++; - } + /* Loop through the doclists in the aaOutput[] array. Merge them all + ** into a single doclist. + */ + for(i=0; iaaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; - *pnKey = (int)(zCsr-z); - zValue = sqlite3_mprintf("%s", &zCsr[1]); - if( zValue ){ - sqlite3Fts3Dequote(zValue); + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } } - *pzValue = zValue; - return 1; + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; } /* -** This function is the implementation of both the xConnect and xCreate -** methods of the FTS3 virtual table. -** -** The argv[] array contains the following: -** -** argv[0] -> module name ("fts3" or "fts4") -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> "column name" and other module argument fields. +** This function is used as the sqlite3Fts3SegReaderIterate() callback when +** querying the full-text index for a doclist associated with a term or +** term-prefix. */ -static int fts3InitVtab( - int isCreate, /* True for xCreate, false for xConnect */ - sqlite3 *db, /* The SQLite database connection */ - void *pAux, /* Hash table containing tokenizers */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ - char **pzErr /* Write any error message here */ +static int fts3TermSelectCb( + Fts3Table *p, /* Virtual table object */ + void *pContext, /* Pointer to TermSelect structure */ + char *zTerm, + int nTerm, + char *aDoclist, + int nDoclist ){ - Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p = 0; /* Pointer to allocated vtab */ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - int nByte; /* Size of allocation used for *p */ - int iCol; /* Column index */ - int nString = 0; /* Bytes required to hold all column names */ - int nCol = 0; /* Number of columns in the FTS table */ - char *zCsr; /* Space for holding column names */ - int nDb; /* Bytes required to hold database name */ - int nName; /* Bytes required to hold table name */ - int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - int bNoDocsize = 0; /* True to omit %_docsize table */ - const char **aCol; /* Array of column names */ - sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - - assert( strlen(argv[0])==4 ); - assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) - || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) - ); + TermSelect *pTS = (TermSelect *)pContext; - nDb = (int)strlen(argv[1]) + 1; - nName = (int)strlen(argv[2]) + 1; + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(zTerm); + UNUSED_PARAMETER(nTerm); - aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) ); - if( !aCol ) return SQLITE_NOMEM; - memset((void *)aCol, 0, sizeof(const char *) * (argc-2)); + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; - /* Loop through all of the arguments passed by the user to the FTS3/4 - ** module (i.e. all the column names and special arguments). This loop - ** does the following: - ** - ** + Figures out the number of columns the FTSX table will have, and - ** the number of bytes of space that must be allocated to store copies - ** of the column names. - ** - ** + If there is a tokenizer specification included in the arguments, - ** initializes the tokenizer pTokenizer. - */ - for(i=3; rc==SQLITE_OK && iaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; - /* Check if this is a tokenizer specification */ - if( !pTokenizer - && strlen(z)>8 - && 0==sqlite3_strnicmp(z, "tokenize", 8) - && 0==sqlite3Fts3IsIdChar(z[8]) - ){ - rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); - } + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } - /* Check if it is an FTS4 special argument. */ - else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ - if( !zVal ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; } - }else{ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; } - sqlite3_free(zVal); } + } + return SQLITE_OK; +} - /* Otherwise, the argument is a column name. */ - else { - nString += (int)(strlen(z) + 1); - aCol[nCol++] = z; +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; } + pCsr->apSegment = apNew; } - if( rc!=SQLITE_OK ) goto fts3_init_out; + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} - if( nCol==0 ){ - assert( nString==0 ); - aCol[0] = "content"; - nString = 8; - nCol = 1; - } +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; + int rc2; + sqlite3_stmt *pStmt = 0; - if( pTokenizer==0 ){ - rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); - if( rc!=SQLITE_OK ) goto fts3_init_out; + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } } - assert( pTokenizer ); - - /* Allocate and populate the Fts3Table structure. */ - nByte = sizeof(Fts3Table) + /* Fts3Table */ - nCol * sizeof(char *) + /* azColumn */ - nName + /* zName */ - nDb + /* zDb */ - nString; /* Space for azColumn strings */ - p = (Fts3Table*)sqlite3_malloc(nByte); - if( p==0 ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } - memset(p, 0, nByte); - p->db = db; - p->nColumn = nCol; - p->nPendingData = 0; - p->azColumn = (char **)&p[1]; - p->pTokenizer = pTokenizer; - p->nNodeSize = 1000; - p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); + } - /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->azColumn[nCol]; - p->zName = zCsr; - memcpy(zCsr, argv[2], nName); - zCsr += nName; - p->zDb = zCsr; - memcpy(zCsr, argv[1], nDb); - zCsr += nDb; + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; - /* Fill in the azColumn array */ - for(iCol=0; iColazColumn[iCol] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); - /* If this is an xCreate call, create the underlying tables in the - ** database. TODO: For xConnect(), it could verify that said tables exist. - */ - if( isCreate ){ - rc = fts3CreateTables(p); + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } } - /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database (see - ** function sqlite3Fts3SegReaderCost() for details). - */ - fts3DatabasePageSize(&rc, p); - - /* Declare the table schema to SQLite. */ - fts3DeclareVtab(&rc, p); - -fts3_init_out: + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; - sqlite3_free((void *)aCol); - if( rc!=SQLITE_OK ){ - if( p ){ - fts3DisconnectMethod((sqlite3_vtab *)p); - }else if( pTokenizer ){ - pTokenizer->pModule->xDestroy(pTokenizer); - } - }else{ - *ppVTab = &p->base; - } return rc; } /* -** The xConnect() and xCreate() methods for the virtual table. All the -** work is done in function fts3InitVtab(). +** Set up a cursor object for iterating through a full-text index or a +** single level therein. */ -static int fts3ConnectMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelaIndex==0 ); + + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); + + return fts3SegReaderCursor( + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + ); } -static int fts3CreateMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ + +static int fts3SegReaderCursorAddZero( + Fts3Table *p, + const char *zTerm, + int nTerm, + Fts3MultiSegReader *pCsr ){ - return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } -/* -** Implementation of the xBestIndex method for FTS3 tables. There -** are three possible strategies, in order of preference: -** -** 1. Direct lookup by rowid or docid. -** 2. Full-text search using a MATCH operator on a non-docid column. -** 3. Linear scan of %_content table. -*/ -static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - Fts3Table *p = (Fts3Table *)pVTab; - int i; /* Iterator variable */ - int iCons = -1; /* Index of constraint to use */ - /* By default use a full table scan. This is an expensive option, - ** so search through the constraints to see if a more efficient - ** strategy is possible. - */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 500000; - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ) continue; +SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +){ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ + int rc = SQLITE_NOMEM; /* Return code */ - /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) - ){ - pInfo->idxNum = FTS3_DOCID_SEARCH; - pInfo->estimatedCost = 1.0; - iCons = i; - } + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); + if( pSegcsr ){ + int i; + int bFound = 0; /* True once an index has been found */ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - /* A MATCH constraint. Use a full-text search. - ** - ** If there is more than one MATCH constraint available, use the first - ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the MATCH strategy. This is done even - ** though the rowid/docid lookup is faster than a MATCH query, selecting - ** it would lead to an "unable to use function MATCH in the requested - ** context" error. - */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH - && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn - ){ - pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; - pInfo->estimatedCost = 2.0; - iCons = i; - break; + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor( + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; } } - if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = 1; - pInfo->aConstraintUsage[iCons].omit = 1; - } - return SQLITE_OK; + *ppSegcsr = pSegcsr; + return rc; +} + +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); } /* -** Implementation of xOpen method. +** This function retreives the doclist for the specified term (or term +** prefix) from the database. +** +** The returned doclist may be in one of two formats, depending on the +** value of parameter isReqPos. If isReqPos is zero, then the doclist is +** a sorted list of delta-compressed docids (a bare doclist). If isReqPos +** is non-zero, then the returned list is in the same format as is stored +** in the database without the found length specifier at the start of on-disk +** doclists. */ -static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int isReqPos, /* True to include position lists in output */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3SegFilter filter; /* Segment term filter configuration */ - UNUSED_PARAMETER(pVTab); + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + tsc.isReqPos = isReqPos; - /* Allocate a buffer large enough for an Fts3Cursor structure. If the - ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, - ** if the allocation fails, return SQLITE_NOMEM. - */ - *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); - if( !pCsr ){ - return SQLITE_NOMEM; + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectCb(p, (void *)&tsc, + pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist + ); } - memset(pCsr, 0, sizeof(Fts3Cursor)); - return SQLITE_OK; + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; } /* -** Close the cursor. For additional information see the documentation -** on the xClose method of the virtual table interface. +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. */ -static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); - sqlite3Fts3ExprFree(pCsr->pExpr); - sqlite3Fts3FreeDeferredTokens(pCsr); - sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); - sqlite3_free(pCsr); - return SQLITE_OK; +static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + if( !isPoslist ){ + /* The number of docids in the list is the same as the number of + ** varints. In FTS3 a varint consists of a single byte with the 0x80 + ** bit cleared and zero or more bytes with the 0x80 bit set. So to + ** count the varints in the buffer, just count the number of bytes + ** with the 0x80 bit clear. */ + while( ppStmt statement so that it is on the row -** of the %_content table that contains the last match. Return -** SQLITE_OK on success. +** Advance the cursor to the next row in the %_content table that +** matches the search criteria. For a MATCH search, this will be +** the next row that matches. For a full-table scan, this will be +** simply the next row in the %_content table. For a docid lookup, +** this routine simply sets the EOF flag. +** +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned +** even if we reach end-of-file. The fts3EofMethod() will be called +** subsequently to determine whether or not an EOF was hit. */ -static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ - if( pCsr->isRequireSeek ){ - pCsr->isRequireSeek = 0; - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - int rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK ){ - /* If no row was found and no error has occured, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. - */ - rc = SQLITE_CORRUPT; - } +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ + int rc; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ pCsr->isEof = 1; - if( pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; } }else{ - return SQLITE_OK; + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; } /* -** This function is used to process a single interior node when searching -** a b-tree for a term or term prefix. The node data is passed to this -** function via the zNode/nNode parameters. The term to search for is -** passed in zTerm/nTerm. +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. ** -** If piFirst is not NULL, then this function sets *piFirst to the blockid -** of the child node that heads the sub-tree that may contain the term. +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. ** -** If piLast is not NULL, then *piLast is set to the right-most child node -** that heads a sub-tree that may contain a term for which zTerm/nTerm is -** a prefix. +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. ** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. */ -static int fts3ScanInteriorNode( - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piFirst, /* OUT: Selected child node */ - sqlite3_int64 *piLast /* OUT: Selected child node */ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ - int isFirstTerm = 1; /* True when processing first term on page */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ + int rc; + char *zSql; /* SQL statement used to access %_content */ + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - /* Skip over the 'height' varint that occurs at the start of every - ** interior node. Then load the blockid of the left-child of the b-tree - ** node into variable iChild. - ** - ** Even if the data structure on disk is corrupted, this (reading two - ** varints from the buffer) does not risk an overread. If zNode is a - ** root node, then the buffer comes from a SELECT statement. SQLite does - ** not make this guarantee explicitly, but in practice there are always - ** either more than 20 bytes of allocated space following the nNode bytes of - ** contents, or two zero bytes. Or, if the node is read from the %_segments - ** table, then there are always 20 bytes of zeroed padding following the - ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). - */ - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - if( zCsr>zEnd ){ - return SQLITE_CORRUPT; + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( nVal==0 || nVal==1 ); + assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + assert( p->pSegments==0 ); + + /* In case the cursor has been used before, clear it now. */ + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; } - - while( zCsrzEnd ){ - rc = SQLITE_CORRUPT; - goto finish_scan; - } - if( nPrefix+nSuffix>nAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - rc = SQLITE_NOMEM; - goto finish_scan; - } - zBuffer = zNew; - } - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; + pCsr->eSearch = (i16)idxNum; - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ - *piFirst = iChild; - piFirst = 0; + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ + int iCol = idxNum-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); + + if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + return SQLITE_NOMEM; } - if( piLast && cmp<0 ){ - *piLast = iChild; - piLast = 0; + rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, + iCol, zQuery, -1, &pCsr->pExpr + ); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_ERROR ){ + static const char *zErr = "malformed MATCH expression: [%s]"; + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); + } + return rc; } - iChild++; - }; + rc = sqlite3Fts3ReadLock(p); + if( rc!=SQLITE_OK ) return rc; - if( piFirst ) *piFirst = iChild; - if( piLast ) *piLast = iChild; + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); - finish_scan: - sqlite3_free(zBuffer); - return rc; -} + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( idxNum==FTS3_FULLSCAN_SEARCH ){ + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); + }else{ + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); + } + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ) return rc; -/* -** The buffer pointed to by argument zNode (size nNode bytes) contains an -** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) -** contains a term. This function searches the sub-tree headed by the zNode -** node for the range of leaf nodes that may contain the specified term -** or terms for which the specified term is a prefix. -** -** If piLeaf is not NULL, then *piLeaf is set to the blockid of the -** left-most leaf node in the tree that may contain the specified term. -** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the -** right-most leaf node that may contain a term for which the specified -** term is a prefix. -** -** It is possible that the range of returned leaf nodes does not contain -** the specified term or any terms for which it is a prefix. However, if the -** segment does contain any such terms, they are stored within the identified -** range. Because this function only inspects interior segment nodes (and -** never loads leaf nodes into memory), it is not possible to be sure. -** -** If an error occurs, an error code other than SQLITE_OK is returned. -*/ -static int fts3SelectLeaf( - Fts3Table *p, /* Virtual table handle */ - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf, /* Selected leaf node */ - sqlite3_int64 *piLeaf2 /* Selected leaf node */ -){ - int rc; /* Return code */ - int iHeight; /* Height of this node in tree */ + if( idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( rc!=SQLITE_OK ) return rc; + } - assert( piLeaf || piLeaf2 ); + return fts3NextMethod(pCursor); +} - sqlite3Fts3GetVarint32(zNode, &iHeight); - rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); - assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + return ((Fts3Cursor *)pCursor)->isEof; +} - if( rc==SQLITE_OK && iHeight>1 ){ - char *zBlob = 0; /* Blob read from %_segments table */ - int nBlob; /* Size of zBlob in bytes */ +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} - if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); - } - sqlite3_free(zBlob); - piLeaf = 0; - zBlob = 0; - } +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+1 ); + + if( iCol==p->nColumn+1 ){ + /* This call is a request for the "docid" column. Since "docid" is an + ** alias for "rowid", use the xRowid() method to obtain the value. + */ + sqlite3_result_int64(pContext, pCsr->iPrevId); + }else if( iCol==p->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor. + */ + sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else{ + rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); - } - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } - sqlite3_free(zBlob); } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + /* -** This function is used to create delta-encoded serialized lists of FTS3 -** varints. Each call to this function appends a single varint to a list. +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. */ -static void fts3PutDeltaVarint( - char **pp, /* IN/OUT: Output pointer */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - sqlite3_int64 iVal /* Write this value to the list */ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + return rc; +} + +/* +** Implementation of xBegin() method. This is a no-op. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c; + + while( p>pStart && (c=*p--)==0 ); + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + if( p>pStart ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ - assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); - *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); - *piPrev = iVal; + Fts3Cursor *pRet; + if( sqlite3_value_type(pVal)!=SQLITE_BLOB + || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + ){ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return SQLITE_ERROR; + } + memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); + *ppCsr = pRet; + return SQLITE_OK; } /* -** When this function is called, *ppPoslist is assumed to point to the -** start of a position-list. After it returns, *ppPoslist points to the -** first byte after the position-list. -** -** A position list is list of positions (delta encoded) and columns for -** a single document record of a doclist. So, in other words, this -** routine advances *ppPoslist so that it points to the next docid in -** the doclist, or to the first byte past the end of the doclist. -** -** If pp is not NULL, then the contents of the position list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. +** Implementation of the snippet() function for FTS3 */ -static void fts3PoslistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ - /* The end of a position list is marked by a zero encoded as an FTS3 - ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by - ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail - ** of some other, multi-byte, value. - ** - ** The following while-loop moves pEnd to point to the first byte that is not - ** immediately preceded by a byte with the 0x80 bit set. Then increments - ** pEnd once more so that it points to the byte immediately following the - ** last byte in the position-list. + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). */ - while( *pEnd | c ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && (*pEnd)==0 ); + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; } - pEnd++; /* Advance past the POS_END terminator byte */ + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + case 5: iCol = sqlite3_value_int(apVal[4]); + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); } - *ppPoslist = pEnd; } /* -** When this function is called, *ppPoslist is assumed to point to the -** start of a column-list. After it returns, *ppPoslist points to the -** to the terminator (POS_COLUMN or POS_END) byte of the column-list. -** -** A column-list is list of delta-encoded positions for a single column -** within a single document within a doclist. -** -** The column-list is terminated either by a POS_COLUMN varint (1) or -** a POS_END varint (0). This routine leaves *ppPoslist pointing to -** the POS_COLUMN or POS_END that terminates the column-list. -** -** If pp is not NULL, then the contents of the column-list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. The POS_COLUMN or POS_END terminator -** is not copied into *pp. +** Implementation of the offsets() function for FTS3 */ -static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - /* A column-list is terminated by either a 0x01 or 0x00 byte that is - ** not part of a multi-byte varint. - */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && ((*pEnd)&0xfe)==0 ); - } - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); } - *ppPoslist = pEnd; } -/* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. -*/ -#define POSITION_LIST_END 0x7fffffff - -/* -** This function is used to help parse position-lists. When this function is -** called, *pp may point to the start of the next varint in the position-list -** being parsed, or it may point to 1 byte past the end of the position-list -** (in which case **pp will be a terminator bytes POS_END (0) or -** (1)). +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: ** -** If *pp points past the end of the current position-list, set *pi to -** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, -** increment the current value of *pi by the value read, and set *pp to -** point to the next value before returning. +** SELECT optimize(t) FROM t LIMIT 1; ** -** Before calling this routine *pi must be initialized to the value of -** the previous position, or zero if we are reading the first position -** in the position-list. Because positions are delta-encoded, the value -** of the previous position is needed in order to compute the value of -** the next position. +** where 't' is the name of an FTS3 table. */ -static void fts3ReadNextPos( - char **pp, /* IN/OUT: Pointer into position-list buffer */ - sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ ){ - if( (**pp)&0xFE ){ - fts3GetDeltaVarint(pp, pi); - *pi -= 2; - }else{ - *pi = POSITION_LIST_END; + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; } } /* -** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by -** the value of iCol encoded as a varint to *pp. This will start a new -** column list. -** -** Set *pp to point to the byte just after the last byte written before -** returning (do not modify it if iCol==0). Return the total number of bytes -** written (0 if iCol==0). +** Implementation of the matchinfo() function for FTS3 */ -static int fts3PutColNumber(char **pp, int iCol){ - int n = 0; /* Number of bytes written */ - if( iCol ){ - char *p = *pp; /* Output pointer */ - n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); - *p = 0x01; - *pp = &p[n]; +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } - return n; } /* -** Compute the union of two position lists. The output written -** into *pp contains all positions of both *pp1 and *pp2 in sorted -** order and with any duplicates removed. All pointers are -** updated appropriately. The caller is responsible for insuring -** that there is enough space in *pp to hold the complete output. +** This routine implements the xFindFunction method for the FTS3 +** virtual table. */ -static void fts3PoslistMerge( - char **pp, /* Output buffer */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ ){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - - while( *p1 || *p2 ){ - int iCol1; /* The current column index in pp1 */ - int iCol2; /* The current column index in pp2 */ - - if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; - else iCol1 = 0; - - if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; - else iCol2 = 0; + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ - if( iCol1==iCol2 ){ - sqlite3_int64 i1 = 0; /* Last position from pp1 */ - sqlite3_int64 i2 = 0; /* Last position from pp2 */ - sqlite3_int64 iPrev = 0; - int n = fts3PutColNumber(&p, iCol1); - p1 += n; - p2 += n; + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); - /* At this point, both p1 and p2 point to the start of column-lists - ** for the same column (the column with index iCol1 and iCol2). - ** A column-list is a list of non-negative delta-encoded varints, each - ** incremented by 2 before being stored. Each list is terminated by a - ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists - ** and writes the results to buffer p. p is left pointing to the byte - ** after the list written. No terminator (POS_END or POS_COLUMN) is - ** written to the output. - */ - fts3GetDeltaVarint(&p1, &i1); - fts3GetDeltaVarint(&p2, &i2); - do { - fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. -** when the *pp1 token appears before the *pp2 token, but not more than nToken -** slots before it. +** Implementation of FTS3 xRename method. Rename an fts3 table. */ -static int fts3PoslistPhraseMerge( - char **pp, /* IN/OUT: Preallocated output buffer */ - int nToken, /* Maximum difference in token positions */ - int isSaveLeft, /* Save the left position */ - int isExact, /* If *pp1 is exactly nTokens before *pp2 */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ +static int fts3RenameMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + const char *zName /* New name of table */ ){ - char *p = (pp ? *pp : 0); - char *p1 = *pp1; - char *p2 = *pp2; - int iCol1 = 0; - int iCol2 = 0; + Fts3Table *p = (Fts3Table *)pVtab; + sqlite3 *db = p->db; /* Database connection */ + int rc; /* Return Code */ - /* Never set both isSaveLeft and isExact for the same invocation. */ - assert( isSaveLeft==0 || isExact==0 ); + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ){ + return rc; + } - assert( *p1!=0 && *p2!=0 ); - if( *p1==POS_COLUMN ){ - p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); } - if( *p2==POS_COLUMN ){ - p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; +} - while( 1 ){ - if( iCol1==iCol2 ){ - char *pSave = p; - sqlite3_int64 iPrev = 0; - sqlite3_int64 iPos1 = 0; - sqlite3_int64 iPos2 = 0; +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + return fts3SyncMethod(pVtab); +} +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} - if( pp && iCol1 ){ - *p++ = POS_COLUMN; - p += sqlite3Fts3PutVarint(p, iCol1); - } +static const sqlite3_module fts3Module = { + /* iVersion */ 2, + /* xCreate */ fts3CreateMethod, + /* xConnect */ fts3ConnectMethod, + /* xBestIndex */ fts3BestIndexMethod, + /* xDisconnect */ fts3DisconnectMethod, + /* xDestroy */ fts3DestroyMethod, + /* xOpen */ fts3OpenMethod, + /* xClose */ fts3CloseMethod, + /* xFilter */ fts3FilterMethod, + /* xNext */ fts3NextMethod, + /* xEof */ fts3EofMethod, + /* xColumn */ fts3ColumnMethod, + /* xRowid */ fts3RowidMethod, + /* xUpdate */ fts3UpdateMethod, + /* xBegin */ fts3BeginMethod, + /* xSync */ fts3SyncMethod, + /* xCommit */ fts3CommitMethod, + /* xRollback */ fts3RollbackMethod, + /* xFindFunction */ fts3FindFunctionMethod, + /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, +}; - assert( *p1!=POS_END && *p1!=POS_COLUMN ); - assert( *p2!=POS_END && *p2!=POS_COLUMN ); - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; +/* +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. +*/ +static void hashDestroy(void *p){ + Fts3Hash *pHash = (Fts3Hash *)p; + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); +} - while( 1 ){ - if( iPos2==iPos1+nToken - || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) - ){ - sqlite3_int64 iSave; - if( !pp ){ - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - return 1; - } - iSave = isSaveLeft ? iPos1 : iPos2; - fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; - pSave = 0; - } - if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ - if( (*p2&0xFE)==0 ) break; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - }else{ - if( (*p1&0xFE)==0 ) break; - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - } - } +/* +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. +** +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifdef SQLITE_ENABLE_ICU +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif - if( pSave ){ - assert( pp && p ); - p = pSave; - } +/* +** Initialise the fts3 extension. If this extension is built as part +** of the sqlite library, then this function is called directly by +** SQLite. If fts3 is built as a dynamically loadable extension, this +** function is called by the sqlite3_extension_init() entry point. +*/ +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + int rc = SQLITE_OK; + Fts3Hash *pHash = 0; + const sqlite3_tokenizer_module *pSimple = 0; + const sqlite3_tokenizer_module *pPorter = 0; - fts3ColumnlistCopy(0, &p1); - fts3ColumnlistCopy(0, &p2); - assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); - if( 0==*p1 || 0==*p2 ) break; +#ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif - p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); - p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; + + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); + + /* Allocate and initialise the hash-table used to store tokenizers. */ + pHash = sqlite3_malloc(sizeof(Fts3Hash)); + if( !pHash ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + } + + /* Load the built-in tokenizers into the hash table */ + if( rc==SQLITE_OK ){ + if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) +#ifdef SQLITE_ENABLE_ICU + || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) +#endif + ){ + rc = SQLITE_NOMEM; } + } - /* Advance pointer p1 or p2 (whichever corresponds to the smaller of - ** iCol1 and iCol2) so that it points to either the 0x00 that marks the - ** end of the position list, or the 0x01 that precedes the next - ** column-number in the position list. - */ - else if( iCol1 D */ -#define MERGE_AND 3 /* D + D -> D */ -#define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ -#define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ -#define MERGE_NEAR 8 /* P + P -> D */ -#define MERGE_POS_NEAR 9 /* P + P -> P */ /* -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, -** which is guaranteed to be large enough to hold the results. The number -** of bytes written to aBuffer is stored in *pnBuffer before returning. +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. ** -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error -** occurs while allocating a temporary buffer as part of the merge operation, -** SQLITE_NOMEM is returned. +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. */ -static int fts3DoclistMerge( - int mergetype, /* One of the MERGE_XXX constants */ - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - char *aBuffer, /* Pre-allocated output buffer */ - int *pnBuffer, /* OUT: Bytes written to aBuffer */ - char *a1, /* Buffer containing first doclist */ - int n1, /* Size of buffer a1 */ - char *a2, /* Buffer containing second doclist */ - int n2, /* Size of buffer a2 */ - int *pnDoc /* OUT: Number of docids in output */ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc ){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - - char *p = aBuffer; - char *p1 = a1; - char *p2 = a2; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - int nDoc = 0; + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR - || mergetype==MERGE_AND || mergetype==MERGE_NOT - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR - ); +static void fts3EvalPhraseMergeToken( + Fts3Table *pTab, + Fts3Phrase *p, + int iToken, + char *pList, + int nList +){ + assert( iToken!=p->iDoclistToken ); - if( !aBuffer ){ - *pnBuffer = 0; - return SQLITE_NOMEM; + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; } - /* Read the first docid from each doclist */ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - - switch( mergetype ){ - case MERGE_OR: - case MERGE_POS_OR: - while( p1 || p2 ){ - if( p2 && p1 && i1==i2 ){ - fts3PutDeltaVarint(&p, &iPrev, i1); - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || (p1 && i1iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } - case MERGE_NOT: - while( p1 ){ - if( p2 && i1==i2 ){ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || i1doclist.aAll==0 ){ + sqlite3_free(pList); + } - case MERGE_POS_PHRASE: - case MERGE_PHRASE: { - char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p); - while( p1 && p2 ){ - if( i1==i2 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - fts3PutDeltaVarint(&p, &iPrev, i1); - if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){ - p = pSave; - iPrev = iPrevSave; - }else{ - nDoc++; - } - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( i1iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; } - default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); { - char *aTmp = 0; - char **ppPos = 0; + fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } - if( mergetype==MERGE_POS_NEAR ){ - ppPos = &p; - aTmp = sqlite3_malloc(2*(n1+n2+1)); - if( !aTmp ){ - return SQLITE_NOMEM; - } - } + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; +} - while( p1 && p2 ){ - if( i1==i2 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - fts3PutDeltaVarint(&p, &iPrev, i1); +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, + Fts3Phrase *p +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; - if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){ - iPrev = iPrevSave; - p = pSave; - } + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( i1pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + if( rc==SQLITE_OK ){ + fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } - sqlite3_free(aTmp); - break; } + assert( pToken->pSegcsr==0 ); } - if( pnDoc ) *pnDoc = nDoc; - *pnBuffer = (int)(p-aBuffer); - return SQLITE_OK; + return rc; } -/* -** A pointer to an instance of this structure is used as the context -** argument to sqlite3Fts3SegReaderIterate() -*/ -typedef struct TermSelect TermSelect; -struct TermSelect { - int isReqPos; - char *aaOutput[16]; /* Malloc'd output buffer */ - int anOutput[16]; /* Size of output in bytes */ -}; +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; + int rc = SQLITE_OK; -/* -** Merge all doclists in the TermSelect.aaOutput[] array into a single -** doclist stored in TermSelect.aaOutput[0]. If successful, delete all -** other doclists (except the aaOutput[0] one) and return SQLITE_OK. -** -** If an OOM error occurs, return SQLITE_NOMEM. In this case it is -** the responsibility of the caller to free any doclists left in the -** TermSelect.aaOutput[] array. -*/ -static int fts3TermSelectMerge(TermSelect *pTS){ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); - char *aOut = 0; - int nOut = 0; - int i; + int nMaxUndeferred = pPhrase->iDoclistToken; + char *aPoslist = 0; + int nPoslist = 0; + int iPrev = -1; + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; - /* Loop through the doclists in the aaOutput[] array. Merge them all - ** into a single doclist. - */ - for(i=0; iaaOutput); i++){ - if( pTS->aaOutput[i] ){ - if( !aOut ){ - aOut = pTS->aaOutput[i]; - nOut = pTS->anOutput[i]; - pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ - sqlite3_free(aOut); - return SQLITE_NOMEM; + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = aOut - aPoslist; + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 - ); - sqlite3_free(pTS->aaOutput[i]); - sqlite3_free(aOut); - pTS->aaOutput[i] = 0; - aOut = aNew; - nOut = nNew; } + iPrev = iToken; } } - pTS->aaOutput[0] = aOut; - pTS->anOutput[0] = nOut; - return SQLITE_OK; -} - -/* -** This function is used as the sqlite3Fts3SegReaderIterate() callback when -** querying the full-text index for a doclist associated with a term or -** term-prefix. -*/ -static int fts3TermSelectCb( - Fts3Table *p, /* Virtual table object */ - void *pContext, /* Pointer to TermSelect structure */ - char *zTerm, - int nTerm, - char *aDoclist, - int nDoclist -){ - TermSelect *pTS = (TermSelect *)pContext; - - UNUSED_PARAMETER(p); - UNUSED_PARAMETER(zTerm); - UNUSED_PARAMETER(nTerm); - - if( pTS->aaOutput[0]==0 ){ - /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist); - pTS->anOutput[0] = nDoclist; - if( pTS->aaOutput[0] ){ - memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + if( iPrev>=0 ){ + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; }else{ - return SQLITE_NOMEM; - } - }else{ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); - char *aMerge = aDoclist; - int nMerge = nDoclist; - int iOut; - - for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; - if( pTS->aaOutput[iOut]==0 ){ - assert( iOut>0 ); - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - break; + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; } - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); - } + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); return SQLITE_NOMEM; } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; } + sqlite3_free(aPoslist); } } - return SQLITE_OK; -} - -static int fts3DeferredTermSelect( - Fts3DeferredToken *pToken, /* Phrase token */ - int isTermPos, /* True to include positions */ - int *pnOut, /* OUT: Size of list */ - char **ppOut /* OUT: Body of list */ -){ - char *aSource; - int nSource; - - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); - if( !aSource ){ - *pnOut = 0; - *ppOut = 0; - }else if( isTermPos ){ - *ppOut = sqlite3_malloc(nSource); - if( !*ppOut ) return SQLITE_NOMEM; - memcpy(*ppOut, aSource, nSource); - *pnOut = nSource; - }else{ - sqlite3_int64 docid; - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); - *ppOut = sqlite3_malloc(*pnOut); - if( !*ppOut ) return SQLITE_NOMEM; - sqlite3Fts3PutVarint(*ppOut, docid); - } return SQLITE_OK; } /* -** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects. -** Elements are added to the array using fts3SegReaderArrayAdd(). -*/ -struct Fts3SegReaderArray { - int nSegment; /* Number of valid entries in apSegment[] */ - int nAlloc; /* Allocated size of apSegment[] */ - int nCost; /* The cost of executing SegReaderIterate() */ - Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */ -}; - - -/* -** Free an Fts3SegReaderArray object. Also free all seg-readers in the -** array (using sqlite3Fts3SegReaderFree()). +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. */ -static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){ - if( pArray ){ - int i; - for(i=0; inSegment; i++){ - sqlite3Fts3SegReaderFree(pArray->apSegment[i]); - } - sqlite3_free(pArray); - } -} +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + int rc; + Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; -static int fts3SegReaderArrayAdd( - Fts3SegReaderArray **ppArray, - Fts3SegReader *pNew -){ - Fts3SegReaderArray *pArray = *ppArray; + if( pCsr->bDesc==pTab->bDescIdx + && bOptOk==1 + && p->nToken==1 + && pFirst->pSegcsr + && pFirst->pSegcsr->bLookup + ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + rc = sqlite3Fts3MsrIncrStart( + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + p->bIncr = 1; - if( !pArray || pArray->nAlloc==pArray->nSegment ){ - int nNew = (pArray ? pArray->nAlloc+16 : 16); - pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, - sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*) - ); - if( !pArray ){ - sqlite3Fts3SegReaderFree(pNew); - return SQLITE_NOMEM; - } - if( nNew==16 ){ - pArray->nSegment = 0; - pArray->nCost = 0; - } - pArray->nAlloc = nNew; - *ppArray = pArray; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; } - pArray->apSegment[pArray->nSegment++] = pNew; - return SQLITE_OK; + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; } -static int fts3TermSegReaderArray( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */ +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int rc; /* Return code */ - Fts3SegReaderArray *pArray = 0; /* Array object to build */ - Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */ - sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ - int iAge = 0; /* Used to assign ages to segments */ + char *p = *ppIter; - /* Allocate a seg-reader to scan the pending terms, if any. */ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader); - if( rc==SQLITE_OK && pReader ) { - rc = fts3SegReaderArrayAdd(&pArray, pReader); - } + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); - /* Loop through the entire %_segdir table. For each segment, create a - ** Fts3SegReader to iterate through the subset of the segment leaves - ** that may contain a term that matches zTerm/nTerm. For non-prefix - ** searches, this is always a single leaf. For prefix searches, this - ** may be a contiguous block of leaves. - */ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, &pStmt); - } - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pNew = 0; - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); - if( sqlite3_column_int64(pStmt, 1)==0 ){ - /* The entire segment is stored on the root node (which must be a - ** leaf). Do not bother inspecting any data in this case, just - ** create a Fts3SegReader to scan the single leaf. - */ - rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew); - }else{ - sqlite3_int64 i1; /* First leaf that may contain zTerm */ - sqlite3_int64 i2; /* Final leaf that may contain zTerm */ - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); - if( isPrefix==0 ) i2 = i1; - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew); - } - } - assert( (pNew==0)==(rc!=SQLITE_OK) ); + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocidnCost); + if( p==aDoclist ){ + *pbEof = 1; + }else{ + char *pSave = p; + fts3ReversePoslist(aDoclist, &p); + *pnList = (pSave - p); } - iAge++; - } - - if( rc==SQLITE_DONE ){ - rc = sqlite3_reset(pStmt); - }else{ - sqlite3_reset(pStmt); + *ppIter = p; } - if( rc!=SQLITE_OK ){ - fts3SegReaderArrayFree(pArray); - pArray = 0; - } - *ppArray = pArray; - return rc; } /* -** This function retreives the doclist for the specified term (or term -** prefix) from the database. +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. ** -** The returned doclist may be in one of two formats, depending on the -** value of parameter isReqPos. If isReqPos is zero, then the doclist is -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos -** is non-zero, then the returned list is in the same format as is stored -** in the database without the found length specifier at the start of on-disk -** doclists. +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. */ -static int fts3TermSelect( - Fts3Table *p, /* Virtual table handle */ - Fts3PhraseToken *pTok, /* Token to query for */ - int iColumn, /* Column to query (or -ve for all columns) */ - int isReqPos, /* True to include position lists in output */ - int *pnOut, /* OUT: Size of buffer at *ppOut */ - char **ppOut /* OUT: Malloced result buffer */ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, + Fts3Phrase *p, + u8 *pbEof ){ - int rc; /* Return code */ - Fts3SegReaderArray *pArray; /* Seg-reader array for this term */ - TermSelect tsc; /* Context object for fts3TermSelectCb() */ - Fts3SegFilter filter; /* Segment term filter configuration */ - - pArray = pTok->pArray; - memset(&tsc, 0, sizeof(TermSelect)); - tsc.isReqPos = isReqPos; - - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY - | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) - | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); - filter.iCol = iColumn; - filter.zTerm = pTok->z; - filter.nTerm = pTok->n; - - rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, - &filter, fts3TermSelectCb, (void *)&tsc - ); - if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); - } + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - if( rc==SQLITE_OK ){ - *ppOut = tsc.aaOutput[0]; - *pnOut = tsc.anOutput[0]; + if( p->bIncr ){ + assert( p->nToken==1 ); + assert( pDL->pNextDocid==0 ); + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( rc==SQLITE_OK && !pDL->pList ){ + *pbEof = 1; + } + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; }else{ - int i; - for(i=0; iaAll[pDL->nAll]; /* 1 byte past end of aAll */ + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; } - } - - fts3SegReaderArrayFree(pArray); - pTok->pArray = 0; - return rc; -} -/* -** This function counts the total number of docids in the doclist stored -** in buffer aList[], size nList bytes. -** -** If the isPoslist argument is true, then it is assumed that the doclist -** contains a position-list following each docid. Otherwise, it is assumed -** that the doclist is simply a list of docids stored as delta encoded -** varints. -*/ -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ - int nDoc = 0; /* Return value */ - if( aList ){ - char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Cursor */ - if( !isPoslist ){ - /* The number of docids in the list is the same as the number of - ** varints. In FTS3 a varint consists of a single byte with the 0x80 - ** bit cleared and zero or more bytes with the 0x80 bit set. So to - ** count the varints in the buffer, just count the number of bytes - ** with the 0x80 bit clear. */ - while( p=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; }else{ - while( pbDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim2(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; } } - return nDoc; + return rc; } -/* -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. -*/ -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pExpr ){ - rc = fts3DeferExpression(pCsr, pExpr->pLeft); - if( rc==SQLITE_OK ){ - rc = fts3DeferExpression(pCsr, pExpr->pRight); - } +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int bOptOk, + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ - int iCol = pExpr->pPhrase->iColumn; int i; - for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; - if( pToken->pDeferred==0 ){ - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); - } + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } - return rc; } -/* -** This function removes the position information from a doclist. When -** called, buffer aList (size *pnList bytes) contains a doclist that includes -** position information. This function removes the position information so -** that aList contains only docids, and adjusts *pnList to reflect the new -** (possibly reduced) size of the doclist. -*/ -static void fts3DoclistStripPositions( - char *aList, /* IN/OUT: Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist in bytes */ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; + int nOvfl; + int iCol; /* The column the token must match */ +}; + +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3Expr *pExpr, + Fts3TokenAndCost **ppTC, + Fts3Expr ***ppOr, + int *pRc ){ - if( aList ){ - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Input cursor */ - char *pOut = aList; /* Output cursor */ - - while( peType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; } -/* -** Return a DocList corresponding to the phrase *pPhrase. -** -** If this function returns SQLITE_OK, but *pnOut is set to a negative value, -** then no tokens in the phrase were looked up in the full-text index. This -** is only possible when this function is called from within xFilter(). The -** caller should assume that all documents match the phrase. The actual -** filtering will take place in xNext(). -*/ -static int fts3PhraseSelect( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - Fts3Phrase *pPhrase, /* Phrase to return a doclist for */ - int isReqPos, /* True if output should contain positions */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut /* OUT: Size of buffer at *paOut */ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3TokenAndCost *aTC, + int nTC ){ - char *pOut = 0; - int nOut = 0; + int nDocSize = 0; + int nDocEst = 0; int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int ii; - int iCol = pPhrase->iColumn; - int isTermPos = (pPhrase->nToken>1 || isReqPos); - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int isFirst = 1; - - int iPrevTok = 0; - int nDoc = 0; - - /* If this is an xFilter() evaluation, create a segment-reader for each - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo - ** evaluation, only create segment-readers if there are no Fts3DeferredToken - ** objects attached to the phrase-tokens. - */ - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pArray==0 ){ - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) - ){ - rc = fts3TermSegReaderArray( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray - ); - if( rc!=SQLITE_OK ) return rc; - } + + int nOvfl = 0; + int nTerm = 0; + + for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok; /* Token to find doclist for */ - int iTok = 0; /* The token being queried this iteration */ - char *pList = 0; /* Pointer to token doclist */ - int nList = 0; /* Size of buffer at pList */ - - /* Select a token to process. If this is an xFilter() call, then tokens - ** are processed in order from least to most costly. Otherwise, tokens - ** are processed in the order in which they occur in the phrase. - */ - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ - assert( isReqPos ); - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - if( pTok->bFulltext==0 ) continue; - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - }else{ - int nMinCost = 0x7FFFFFFF; - int jj; + rc = fts3EvalAverageDocsize(pCsr, &nDocSize); - /* Find the remaining token with the lowest cost. */ - for(jj=0; jjnToken; jj++){ - Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray; - if( pArray && pArray->nCostnCost; - } - } - pTok = &pPhrase->aToken[iTok]; + for(ii=0; iinDoc && ii>0 ){ - rc = fts3DeferExpression(pCsr, pCsr->pExpr); - break; + for(jj=0; jjnOvfl) + ){ + pTC = &aTC[jj]; } } + assert( pTC ); - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); - }else{ - if( pTok->pArray ){ - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); - } - pTok->bFulltext = 1; - } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 ); - if( rc!=SQLITE_OK ) break; + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); - if( isFirst ){ - pOut = pList; - nOut = nList; - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ - nDoc = fts3DoclistCountDocids(1, pOut, nOut); + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + } } - isFirst = 0; - iPrevTok = iTok; }else{ - /* Merge the new term list and the current output. */ - char *aLeft, *aRight; - int nLeft, nRight; - int nDist; - int mt; - - /* If this is the final token of the phrase, and positions were not - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. - ** This drops the position information from the output list. - */ - mt = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; - - assert( iPrevTok!=iTok ); - if( iPrevToknOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); } - assert( nOut==0 || pOut!=0 ); + pTC->pToken = 0; } - if( rc==SQLITE_OK ){ - if( ii!=pPhrase->nToken ){ - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); - fts3DoclistStripPositions(pOut, &nOut); - } - *paOut = pOut; - *pnOut = nOut; - }else{ - sqlite3_free(pOut); - } return rc; } -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ +SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: + ** This call will eventually also be responsible for determining which + ** tokens are 'deferred' until the document text is loaded into memory. + ** + ** Each token in each phrase is dealt with using one of the following + ** three strategies: + ** + ** 1. Entire doclist loaded into memory as part of the + ** fts3EvalStartReaders() call. + ** + ** 2. Doclist loaded into memory incrementally, as part of each + ** sqlite3Fts3EvalNext() call. + ** + ** 3. Token doclist is never loaded. Instead, documents are loaded into + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() + ** call. This is known as a "deferred" token. + */ - *paOut = aOut; - return rc; -} + /* If bOptOk is true, check if there are any tokens that should be deferred. + */ + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; -/* -** This function is used as part of the processing for the snippet() and -** offsets() functions. -** -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both -** have their respective doclists (including position information) loaded -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from -** each doclist that are not within nNear tokens of a corresponding entry -** in the other doclist. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ - int rc; /* Return code */ + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; - assert( pLeft->eType==FTSQUERY_PHRASE ); - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->isLoaded && pRight->isLoaded ); + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + nToken = pTC-aTC; + nOr = ppOr-apOr; - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ - sqlite3_free(pLeft->aDoclist); - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = 0; - pLeft->aDoclist = 0; - rc = SQLITE_OK; - }else{ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - &aOut, &nOut - ); - if( rc!=SQLITE_OK ) return rc; - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = aOut; - pRight->nDoclist = nOut; - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - &aOut, &nOut - ); - sqlite3_free(pLeft->aDoclist); - pLeft->aDoclist = aOut; - pLeft->nDoclist = nOut; + sqlite3_free(aTC); + } } + + fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); return rc; } +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} -/* -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. -** The allocated objects are stored in the Fts3PhraseToken.pArray member -** variables of each token structure. -*/ -static int fts3ExprAllocateSegReaders( - Fts3Cursor *pCsr, /* FTS3 table */ - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - int *pnExpr /* OUT: Number of AND'd expressions */ +static int fts3EvalNearTrim2( + int nNear, + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ ){ - int rc = SQLITE_OK; /* Return code */ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ - (*pnExpr)++; - pnExpr = 0; + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; } - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; + return res; +} + +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pArray==0 ){ - rc = fts3TermSegReaderArray( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear = p->pParent->nNear; + Fts3Phrase *pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase ); + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); } } - }else{ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); - if( rc==SQLITE_OK ){ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); - } - } - return rc; -} -/* -** Free the Fts3SegReaderArray objects associated with each token in the -** expression pExpr. In other words, this function frees the resources -** allocated by fts3ExprAllocateSegReaders(). -*/ -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase ){ - int kk; - for(kk=0; kknToken; kk++){ - fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray); - pPhrase->aToken[kk].pArray = 0; - } - } - fts3ExprFreeSegReaders(pExpr->pLeft); - fts3ExprFreeSegReaders(pExpr->pRight); + sqlite3_free(aTmp); } -} -/* -** Return the sum of the costs of all tokens in the expression pExpr. This -** function must be called after Fts3SegReaderArrays have been allocated -** for all tokens using fts3ExprAllocateSegReaders(). -*/ -static int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray; - if( pArray ){ - nCost += pPhrase->aToken[ii].pArray->nCost; - } - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; + return res; } /* -** The following is a helper function (and type) for fts3EvalExpr(). It -** must be called after Fts3SegReaders have been allocated for every token -** in the expression. See the context it is called from in fts3EvalExpr() -** for further explanation. +** This macro is used by the fts3EvalNext() function. The two arguments are +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then +** it returns (i2 - i1). This allows the same code to be used for merging +** doclists in ascending or descending order. */ -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr); - (*ppExprCost)++; - } -} +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) -/* -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. -** -** This function is called in two distinct contexts: -** -** * From within the virtual table xFilter() method. In this case, the -** output doclist contains entries for all rows in the table, based on -** data read from the full-text index. -** -** In this case, if the query expression contains one or more tokens that -** are very common, then the returned doclist may contain a superset of -** the documents that actually match the expression. -** -** * From within the virtual table xNext() method. This call is only made -** if the call from within xFilter() found that there were very common -** tokens in the query expression and did return a superset of the -** matching documents. In this case the returned doclist contains only -** entries that correspond to the current row of the table. Instead of -** reading the data for each token from the full-text index, the data is -** already available in-memory in the Fts3PhraseToken.pDeferred structures. -** See fts3EvalDeferred() for how it gets there. -** -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is -** required) Fts3Cursor.doDeferred==1. -** -** If the SQLite invokes the snippet(), offsets() or matchinfo() function -** as part of a SELECT on an FTS3 table, this function is called on each -** individual phrase expression in the query. If there were very common tokens -** found in the xFilter() call, then this function is called once for phrase -** for each row visited, and the returned doclist contains entries for the -** current row only. Otherwise, if there were no very common tokens, then this -** function is called once only for each phrase in the query and the returned -** doclist contains entries for all rows of the table. -** -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a -** result of a snippet(), offsets() or matchinfo() invocation. -*/ -static int fts3EvalExpr( - Fts3Cursor *p, /* Virtual table cursor handle */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut, /* OUT: Size of buffer at *paOut */ - int isReqPos /* Require positions in output buffer */ +static void fts3EvalNext( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc ){ - int rc = SQLITE_OK; /* Return code */ + if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + if( pLeft->bDeferred ){ + fts3EvalNext(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + fts3EvalNext(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + } - /* Zero the output parameters. */ - *paOut = 0; - *pnOut = 0; + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNext(pCsr, pRight, pRc); + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + } - if( pExpr ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - || pExpr->eType==FTSQUERY_PHRASE - ); - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } - if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), - paOut, pnOut - ); - fts3ExprFreeSegReaders(pExpr); - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ - int nExpr = 0; /* Size of aExpr[] */ - char *aRet = 0; /* Doclist to return to caller */ - int nRet = 0; /* Length of aRet[] in bytes */ - int nDoc = 0x7FFFFFFF; + break; + } - assert( !isReqPos ); + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); - if( rc==SQLITE_OK ){ - assert( nExpr>1 ); - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); - if( !aExpr ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - int ii; /* Used to iterate through expressions */ - - fts3ExprAssignCosts(pExpr, &aExpr); - aExpr -= nExpr; - for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ - pBest = pCand; - } - } - - if( pBest->nCost>nDoc ){ - rc = fts3DeferExpression(p, p->pExpr); - break; - }else{ - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); - if( rc!=SQLITE_OK ) break; - pBest->pExpr = 0; - if( ii==0 ){ - aRet = aNew; - nRet = nNew; - nDoc = fts3DoclistCountDocids(0, aRet, nRet); - }else{ - fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc - ); - sqlite3_free(aNew); - } + if( pRight->bStart==0 ){ + fts3EvalNext(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNext(pCsr, pRight, pRc); } } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; } - if( rc==SQLITE_OK ){ - *paOut = aRet; - *pnOut = nRet; - }else{ - assert( *paOut==0 ); - sqlite3_free(aRet); + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalZeroPoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; } - sqlite3_free(aExpr); - fts3ExprFreeSegReaders(pExpr); + } + } +} - }else{ - char *aLeft; - char *aRight; - int nLeft; - int nRight; - - assert( pExpr->eType==FTSQUERY_NEAR - || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_NOT - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) - ); +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + int bHit = 1; + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) - ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: { - Fts3Expr *pLeft; - Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - mergetype = MERGE_POS_NEAR; - } - pLeft = pExpr->pLeft; - while( pLeft->eType==FTSQUERY_NEAR ){ - pLeft=pLeft->pRight; + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); } - pRight = pExpr->pRight; - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - - rc = fts3NearMerge(mergetype, pExpr->nNear, - pLeft->pPhrase->nToken, aLeft, nLeft, - pRight->pPhrase->nToken, aRight, nRight, - paOut, pnOut - ); - sqlite3_free(aLeft); - break; } - - case FTSQUERY_OR: { - /* Allocate a buffer for the output. The maximum size is the - ** sum of the sizes of the two input buffers. The +1 term is - ** so that a buffer of zero bytes is never allocated - this can - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. - */ - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aBuffer; - sqlite3_free(aLeft); - break; + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); } + } - default: { - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aLeft; - break; + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else{ + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); } + break; } - sqlite3_free(aRight); } } - - assert( rc==SQLITE_OK || *paOut==0 ); - return rc; + return bHit; } /* -** This function is called from within xNext() for each row visited by -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() -** was able to determine the exact set of matching rows, this function sets -** *pbRes to true and returns SQLITE_IO immediately. +** Return 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and ** -** Otherwise, if evaluating the query expression within xFilter() returned a -** superset of the matching documents instead of an exact set (this happens -** when the query includes very common tokens and it is deemed too expensive to -** load their doclists from disk), this function tests if the current row -** really does match the FTS3 query. +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does not match the query. ** -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK -** is returned and *pbRes is set to true if the current row matches the -** FTS3 query (and should be included in the results returned to SQLite), or -** false otherwise. +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. */ -static int fts3EvalDeferred( - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ - int *pbRes /* OUT: Set to true if row is a match */ -){ - int rc = SQLITE_OK; - if( pCsr->pDeferred==0 ){ - *pbRes = 1; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - if( rc==SQLITE_OK ){ - char *a = 0; - int n = 0; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); - assert( n>=0 ); - *pbRes = (n>0); - sqlite3_free(a); +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } } + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; } - return rc; + return (rc==SQLITE_OK && bMiss); } /* -** Advance the cursor to the next row in the %_content table that -** matches the search criteria. For a MATCH search, this will be -** the next row that matches. For a full-table scan, this will be -** simply the next row in the %_content table. For a docid lookup, -** this routine simply sets the EOF flag. -** -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned -** even if we reach end-of-file. The fts3EofMethod() will be called -** subsequently to determine whether or not an EOF was hit. +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. */ -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int res; - int rc = SQLITE_OK; /* Return code */ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - }else{ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; +SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); } - sqlite3_reset(pCsr->pStmt); - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNext(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; - } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); - + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + } return rc; } /* -** This is the xFilter interface for the virtual table. See -** the virtual table xFilter method documentation for additional -** information. -** -** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against -** the %_content table. +** Restart interation for expression pExpr so that the next call to +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. ** -** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry -** in the %_content table. -** -** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The -** column on the left-hand side of the MATCH operator is column -** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand -** side of the MATCH operator. +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. */ -static int fts3FilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc ){ - const char *azSql[] = { - "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */ - "SELECT * FROM %Q.'%q_content'", /* full-table-scan */ - }; - int rc; /* Return code */ - char *zSql; /* SQL statement used to access %_content */ - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); - - assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( nVal==0 || nVal==1 ); - assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); - assert( p->pSegments==0 ); - - /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->aDoclist); - sqlite3Fts3ExprFree(pCsr->pExpr); - memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - - if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ - int iCol = idxNum-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); - - if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - return SQLITE_NOMEM; - } + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, - iCol, zQuery, -1, &pCsr->pExpr - ); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_ERROR ){ - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", - zQuery); + if( pPhrase ){ + fts3EvalZeroPoslist(pPhrase); + if( pPhrase->bIncr ){ + assert( pPhrase->nToken==1 ); + assert( pPhrase->aToken[0].pSegcsr ); + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } - return rc; - } - rc = sqlite3Fts3ReadLock(p); - if( rc!=SQLITE_OK ) return rc; + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); - sqlite3Fts3SegmentsClose(p); - if( rc!=SQLITE_OK ) return rc; - pCsr->pNextId = pCsr->aDoclist; - pCsr->iPrevId = 0; - } + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); } - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - } - pCsr->eSearch = (i16)idxNum; - - if( rc!=SQLITE_OK ) return rc; - return fts3NextMethod(pCursor); } -/* -** This is the xEof method of the virtual table. SQLite calls this -** routine to find out if it has reached the end of a result set. +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. */ -static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ - return ((Fts3Cursor *)pCursor)->isEof; -} +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; -/* -** This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The -** rowid should be written to *pRowid. -*/ -static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->aDoclist ){ - *pRowid = pCsr->iPrevId; - }else{ - /* This branch runs if the query is implemented using a full-table scan - ** (not using the full-text index). In this case grab the rowid from the - ** SELECT statement. - */ - assert( pCsr->isRequireSeek==0 ); - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += sqlite3Fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); } - return SQLITE_OK; } -/* -** This is the xColumn method, called by SQLite to request a value from -** the row that the supplied cursor currently points to. +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. */ -static int fts3ColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ ){ - int rc; /* Return Code */ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; + int rc = SQLITE_OK; /* Return code */ - /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+1 ); + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. - */ - sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; } - } - return rc; -} + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); -/* -** This function is the implementation of the xUpdate callback used by -** FTS3 virtual tables. It is invoked by SQLite each time a row is to be -** inserted, updated or deleted. -*/ -static int fts3UpdateMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Size of argument array */ - sqlite3_value **apVal, /* Array of arguments */ - sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ -){ - return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); -} + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } -/* -** Implementation of xSync() method. Flush the contents of the pending-terms -** hash-table to the database. -*/ -static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); - return rc; -} + fts3EvalRestart(pCsr, pRoot, &rc); -/* -** Implementation of xBegin() method. This is a no-op. -*/ -static int fts3BeginMethod(sqlite3_vtab *pVtab){ - UNUSED_PARAMETER(pVtab); - assert( ((Fts3Table *)pVtab)->nPendingData==0 ); - return SQLITE_OK; -} + while( pCsr->isEof==0 && rc==SQLITE_OK ){ -/* -** Implementation of xCommit() method. This is a no-op. The contents of -** the pending-terms hash-table have already been flushed into the database -** by fts3SyncMethod(). -*/ -static int fts3CommitMethod(sqlite3_vtab *pVtab){ - UNUSED_PARAMETER(pVtab); - assert( ((Fts3Table *)pVtab)->nPendingData==0 ); - return SQLITE_OK; -} + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNext(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalLoadDeferred(pCsr, &rc) + ); -/* -** Implementation of xRollback(). Discard the contents of the pending-terms -** hash-table. Any changes made to the database are reverted by SQLite. -*/ -static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); - return SQLITE_OK; -} + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } -/* -** Load the doclist associated with expression pExpr to pExpr->aDoclist. -** The loaded doclist contains positions as well as the document ids. -** This is used by the matchinfo(), snippet() and offsets() auxillary -** functions. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); - return rc; -} + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; -SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - char **paDoclist, - int *pnDoclist -){ - int rc; - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); - pCsr->eEvalmode = FTS3_EVAL_NEXT; + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + fts3EvalLoadDeferred(pCsr, &rc); + } + } return rc; } /* -** After ExprLoadDoclist() (see above) has been called, this function is -** used to iterate/search through the position lists that make up the doclist -** stored in pExpr->aDoclist. +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. */ -SQLITE_PRIVATE char *sqlite3Fts3FindPositions( - Fts3Expr *pExpr, /* Access this expressions doclist */ - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ - int iCol /* Column of requested pos-list */ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ ){ - assert( pExpr->isLoaded ); - if( pExpr->aDoclist ){ - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - char *pCsr = pExpr->pCurrent; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; - assert( pCsr ); - while( pCsriCurrentiCurrent); - } - pExpr->pCurrent = pCsr; - }else{ - if( pExpr->iCurrent==iDocid ){ - int iThis = 0; - if( iCol<0 ){ - /* If iCol is negative, return a pointer to the start of the - ** position-list (instead of a pointer to the start of a list - ** of offsets associated with a specific column). - */ - return pCsr; - } - while( iThisbDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; } } } - return 0; + return rc; } /* -** Helper function used by the implementation of the overloaded snippet(), -** offsets() and optimize() SQL functions. +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. ** -** If the value passed as the third argument is a blob of size -** sizeof(Fts3Cursor*), then the blob contents are copied to the -** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error -** message is written to context pContext and SQLITE_ERROR returned. The -** string passed via zFunc is used as part of the error message. +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. */ -static int fts3FunctionArg( - sqlite3_context *pContext, /* SQL function call context */ - const char *zFunc, /* Function name */ - sqlite3_value *pVal, /* argv[0] passed to function */ - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol /* Column to return position list for */ ){ - Fts3Cursor *pRet; - if( sqlite3_value_type(pVal)!=SQLITE_BLOB - || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter = pPhrase->doclist.pList; + int iThis; + + assert( iCol>=0 && iColnColumn ); + if( !pIter + || pExpr->bEof + || pExpr->iDocid!=pCsr->iPrevId + || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ - char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); - sqlite3_result_error(pContext, zErr, -1); - sqlite3_free(zErr); - return SQLITE_ERROR; + return 0; } - memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); - *ppCsr = pRet; - return SQLITE_OK; + + assert( pPhrase->doclist.nList>0 ); + if( *pIter==0x01 ){ + pIter++; + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. */ -static void fts3SnippetFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of apVal[] array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - int iCol = -1; - int nToken = 15; /* Default number of tokens in snippet */ +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalZeroPoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} - /* There must be at least one argument passed to this function (otherwise - ** the non-overloaded version would have been called instead of this one). - */ - assert( nVal>=1 ); +#endif - if( nVal>6 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function snippet()", -1); - return; - } - if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; +/************** End of fts3.c ************************************************/ +/************** Begin file fts3_aux.c ****************************************/ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; - switch( nVal ){ - case 6: nToken = sqlite3_value_int(apVal[5]); - case 5: iCol = sqlite3_value_int(apVal[4]); - case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); - case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); - case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); - } - if( !zEllipsis || !zEnd || !zStart ){ - sqlite3_result_error_nomem(pContext); - }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); - } -} +/* +** Schema of the terms table. +*/ +#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)" /* -** Implementation of the offsets() function for FTS3 +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. */ -static void fts3OffsetsFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ - UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(pUnused); - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; - assert( pCsr ); - if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Offsets(pContext, pCsr); + /* The user should specify a single argument - the name of an fts3 table. */ + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "wrong number of arguments to fts4aux constructor" + ); + return SQLITE_ERROR; } -} -/* -** Implementation of the special optimize() function for FTS3. This -** function merges all segments in the database to a single segment. -** Example usage is: -** -** SELECT optimize(t) FROM t LIMIT 1; -** -** where 't' is the name of an FTS3 table. -*/ -static void fts3OptimizeFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - int rc; /* Return code */ - Fts3Table *p; /* Virtual table handle */ - Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + zDb = argv[1]; + nDb = strlen(zDb); + zFts3 = argv[3]; + nFts3 = strlen(zFts3); - UNUSED_PARAMETER(nVal); + rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + if( rc!=SQLITE_OK ) return rc; - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; - p = (Fts3Table *)pCursor->base.pVtab; - assert( p ); + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); - rc = sqlite3Fts3Optimize(p); + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; - switch( rc ){ - case SQLITE_OK: - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - break; - case SQLITE_DONE: - sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); - break; - default: - sqlite3_result_error_code(pContext, rc); - break; - } + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; } /* -** Implementation of the matchinfo() function for FTS3 +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. */ -static void fts3MatchinfoFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - assert( nVal==1 || nVal==2 ); - if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - const char *zArg = 0; - if( nVal>1 ){ - zArg = (const char *)sqlite3_value_text(apVal[1]); - } - sqlite3Fts3Matchinfo(pContext, pCsr, zArg); +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; } +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + /* -** This routine implements the xFindFunction method for the FTS3 -** virtual table. +** xBestIndex - Analyze a WHERE and ORDER BY clause. */ -static int fts3FindFunctionMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Number of SQL function arguments */ - const char *zName, /* Name of SQL function */ - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ - void **ppArg /* Unused */ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo ){ - struct Overloaded { - const char *zName; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aOverload[] = { - { "snippet", fts3SnippetFunc }, - { "offsets", fts3OffsetsFunc }, - { "optimize", fts3OptimizeFunc }, - { "matchinfo", fts3MatchinfoFunc }, - }; - int i; /* Iterator variable */ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; - UNUSED_PARAMETER(pVtab); - UNUSED_PARAMETER(nArg); - UNUSED_PARAMETER(ppArg); + UNUSED_PARAMETER(pVTab); - for(i=0; inOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){ + int op = pInfo->aConstraint[i].op; + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; } } - /* No function of the specified name was found. Return 0. */ - return 0; + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = 1; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = 1; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0); + pInfo->estimatedCost /= 2; + } + } + + return SQLITE_OK; } /* -** Implementation of FTS3 xRename method. Rename an fts3 table. +** xOpen - Open a cursor. */ -static int fts3RenameMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - const char *zName /* New name of table */ -){ - Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; /* Database connection */ - int rc; /* Return Code */ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ - rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ){ - return rc; - } + UNUSED_PARAMETER(pVTab); - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", - p->zDb, p->zName, zName + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize ); - } - if( p->bHasStat ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", - p->zDb, p->zName, zName + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", - p->zDb, p->zName, zName - ); - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", - p->zDb, p->zName, zName - ); - return rc; + return SQLITE_OK; } -static const sqlite3_module fts3Module = { - /* iVersion */ 0, - /* xCreate */ fts3CreateMethod, - /* xConnect */ fts3ConnectMethod, - /* xBestIndex */ fts3BestIndexMethod, - /* xDisconnect */ fts3DisconnectMethod, - /* xDestroy */ fts3DestroyMethod, - /* xOpen */ fts3OpenMethod, - /* xClose */ fts3CloseMethod, - /* xFilter */ fts3FilterMethod, - /* xNext */ fts3NextMethod, - /* xEof */ fts3EofMethod, - /* xColumn */ fts3ColumnMethod, - /* xRowid */ fts3RowidMethod, - /* xUpdate */ fts3UpdateMethod, - /* xBegin */ fts3BeginMethod, - /* xSync */ fts3SyncMethod, - /* xCommit */ fts3CommitMethod, - /* xRollback */ fts3RollbackMethod, - /* xFindFunction */ fts3FindFunctionMethod, - /* xRename */ fts3RenameMethod, -}; - /* -** This function is registered as the module destructor (called when an -** FTS3 enabled database connection is closed). It frees the memory -** allocated for the tokenizer hash table. +** xNext - Advance the cursor to the next row, if any. */ -static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + } + return rc; } /* -** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are -** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c -** respectively. The following three forward declarations are for functions -** declared in these files used to retrieve the respective implementations. -** -** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point to the "simple" tokenizer implementation. -** And so on. +** xFilter - Initialize a cursor to point at the start of its data. */ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifdef SQLITE_ENABLE_ICU -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#endif +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan; -/* -** Initialise the fts3 extension. If this extension is built as part -** of the sqlite library, then this function is called directly by -** SQLite. If fts3 is built as a dynamically loadable extension, this -** function is called by the sqlite3_extension_init() entry point. -*/ -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ - int rc = SQLITE_OK; - Fts3Hash *pHash = 0; - const sqlite3_tokenizer_module *pSimple = 0; - const sqlite3_tokenizer_module *pPorter = 0; + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); -#ifdef SQLITE_ENABLE_ICU - const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts3IcuTokenizerModule(&pIcu); -#endif + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT); - sqlite3Fts3SimpleTokenizerModule(&pSimple); - sqlite3Fts3PorterTokenizerModule(&pPorter); + /* In case this cursor is being reused, close and zero it. */ + testcase(pCsr->filter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); - /* Allocate and initialise the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); - if( !pHash ){ - rc = SQLITE_NOMEM; - }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); - } + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - /* Load the built-in tokenizers into the hash table */ - if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) -#ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) -#endif - ){ - rc = SQLITE_NOMEM; + if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; } } + if( idxNum&FTS4AUX_LE_CONSTRAINT ){ + int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0; + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); + pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + } -#ifdef SQLITE_TEST + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db); + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); } -#endif - /* Create the virtual table wrapper around the hash-table and overload - ** the two scalar functions. If this is successful, register the - ** module with sqlite. - */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) - ){ - rc = sqlite3_create_module_v2( - db, "fts3", &fts3Module, (void *)pHash, hashDestroy - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 - ); + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + if( iCol==0 ){ /* Column "term" */ + sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + }else if( iCol==1 ){ /* Column "col" */ + if( p->iCol ){ + sqlite3_result_int(pContext, p->iCol-1); + }else{ + sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC); } - return rc; + }else if( iCol==2 ){ /* Column "documents" */ + sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc); + }else{ /* Column "occurrences" */ + sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc); } - /* An error has occurred. Delete the hash table and return the error code. */ - assert( rc!=SQLITE_OK ); - if( pHash ){ - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); - } - return rc; + return SQLITE_OK; } -#if !SQLITE_CORE -SQLITE_API int sqlite3_extension_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ ){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; } -#endif -#endif +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ -/************** End of fts3.c ************************************************/ + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_aux.c ********************************************/ /************** Begin file fts3_expr.c ***************************************/ /* ** 2008 Nov 28 @@ -111802,12 +116992,21 @@ SQLITE_API int sqlite3_fts3_enable_parentheses = 0; #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; @@ -111893,7 +117092,7 @@ static int getNextToken( iEnd++; } if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pRet->pPhrase->isNot = 1; + pParse->isNot = 1; } } nConsumed = iEnd; @@ -111945,36 +117144,55 @@ static int getNextString( char *zTemp = 0; int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ - const char *zToken; - int nToken, iBegin, iEnd, iPos; - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + const char *zByte; + int nByte, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); - if( !p || !zTemp ){ - goto no_mem; - } - if( ii==0 ){ - memset(p, 0, nByte); - p->pPhrase = (Fts3Phrase *)&p[1]; - } - p->pPhrase = (Fts3Phrase *)&p[1]; - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); - p->pPhrase->nToken = ii+1; - p->pPhrase->aToken[ii].n = nToken; - memcpy(&zTemp[nTemp], zToken, nToken); - nTemp += nToken; - if( iEndpPhrase->aToken[ii].isPrefix = 1; - }else{ - p->pPhrase->aToken[ii].isPrefix = 0; - } + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndpPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); - p = fts3ReallocOrFree(p, nByte + nTemp); - if( !p ){ - goto no_mem; - } - if( zTemp ){ - zNew = &(((char *)p)[nByte]); - memcpy(zNew, zTemp, nTemp); - }else{ - memset(p, 0, nByte+nTemp); - } + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = &zNew[nNew]; - nNew += p->pPhrase->aToken[jj].n; + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; } - sqlite3_free(zTemp); - p->eType = FTSQUERY_PHRASE; - p->pPhrase->iColumn = pParse->iDefaultCol; rc = SQLITE_OK; } @@ -112062,6 +117276,8 @@ static int getNextNode( const char *zInput = z; int nInput = n; + pParse->isNot = 0; + /* Skip over any whitespace before checking for a keyword, an open or ** close bracket, or a quoted string. */ @@ -112280,7 +117496,7 @@ static int fts3ExprParse( int isPhrase; if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); @@ -112298,7 +117514,6 @@ static int fts3ExprParse( p = pPrev; }else{ int eType = p->eType; - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); /* The isRequirePhrase variable is set to true if a phrase or @@ -112461,9 +117676,11 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( */ SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - sqlite3_free(p->aDoclist); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); sqlite3_free(p); } } @@ -112520,7 +117737,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; zBuf = sqlite3_mprintf( - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, @@ -113067,7 +118284,6 @@ SQLITE_PRIVATE void *sqlite3Fts3HashInsert( - /* ** Class derived from sqlite3_tokenizer */ @@ -113707,12 +118923,12 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + /* ** Implementation of the SQL scalar function for accessing the underlying @@ -113836,7 +119052,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( ){ int rc; char *z = (char *)zArg; - int n; + int n = 0; char *zCopy; char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; @@ -114201,7 +119417,6 @@ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( - typedef struct simple_tokenizer { sqlite3_tokenizer base; char delim[128]; /* flag ASCII delimiters */ @@ -114438,14 +119653,40 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( */ #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; @@ -114476,7 +119717,6 @@ struct Fts3DeferredToken { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -114495,6 +119735,9 @@ struct Fts3SegReader { char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -114508,8 +119751,11 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; @@ -114547,6 +119793,14 @@ struct SegmentWriter { ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ @@ -114577,10 +119831,10 @@ struct SegmentNode { #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 @@ -114589,6 +119843,11 @@ struct SegmentNode { #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 + +#define SQL_DELETE_SEGDIR_RANGE 26 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -114614,7 +119873,7 @@ static int fts3SqlStmt( /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", +/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", @@ -114624,19 +119883,25 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", -/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -114648,20 +119913,9 @@ static int fts3SqlStmt( if( !pStmt ){ char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ - int i; /* Iterator variable */ - char *zVarlist; /* The "?, ?, ..." string */ - zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2); - if( !zVarlist ){ - *pp = 0; - return SQLITE_NOMEM; - } - zVarlist[0] = '?'; - zVarlist[p->nColumn*2+1] = '\0'; - for(i=1; i<=p->nColumn; i++){ - zVarlist[i*2-1] = ','; - zVarlist[i*2] = '?'; - } - zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist); + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } @@ -114702,9 +119956,9 @@ static int fts3SelectDocsize( sqlite3_bind_int64(pStmt, 1, iDocid); } rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ){ + if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; + if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT_VTAB; pStmt = 0; }else{ rc = SQLITE_OK; @@ -114803,8 +120057,35 @@ SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ ** 3: end_block ** 4: root */ -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){ - return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + } + } + *ppStmt = pStmt; + return rc; } @@ -114916,6 +120197,47 @@ static int fts3PendingListAppend( return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -114946,6 +120268,14 @@ static int fts3PendingTermsAdd( assert( pTokenizer && pModule ); + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; @@ -114956,8 +120286,7 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ - PendingList *pList; - + int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -114968,22 +120297,19 @@ static int fts3PendingTermsAdd( break; } - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); } } @@ -115013,14 +120339,19 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ } /* -** Discard the contents of the pending-terms hash table. +** Discard the contents of the pending-terms hash tables. */ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); } - fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } @@ -115036,11 +120367,9 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); - if( zText ){ - int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); + if( rc!=SQLITE_OK ){ + return rc; } aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } @@ -115145,14 +120474,14 @@ static int fts3DeleteAll(Fts3Table *p){ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ - sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ + sqlite3_value *pRowid, /* The docid to be deleted */ u32 *aSz /* Sizes of deleted document written here */ ){ int rc; sqlite3_stmt *pSelect; if( *pRC ) return; - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; @@ -115178,7 +120507,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int); +static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -115195,7 +120524,12 @@ static int fts3SegmentMerge(Fts3Table *, int); ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ @@ -115203,7 +120537,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iLevel); + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -115217,7 +120551,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iLevel); + rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -115258,7 +120592,8 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob /* OUT: Size of blob data */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ ){ int rc; /* Return code */ @@ -115279,11 +120614,16 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; if( paBlob ){ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); memset(&aByte[nByte], 0, FTS3_NODE_PADDING); if( rc!=SQLITE_OK ){ @@ -115293,7 +120633,6 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( } *paBlob = aByte; } - *pnBlob = nByte; } return rc; @@ -115307,13 +120646,55 @@ SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -115325,7 +120706,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -115345,6 +120725,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; } pReader->aNode = 0; @@ -115356,21 +120738,31 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } pNext = pReader->aNode; } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } if( nPrefix+nSuffix>pReader->nTermAlloc ){ @@ -115382,6 +120774,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; @@ -115394,9 +120790,9 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** of these statements is untrue, then the data structure is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || pReader->aDoclist[pReader->nDoclist-1] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } return SQLITE_OK; } @@ -115405,12 +120801,26 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int n; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; } /* @@ -115423,122 +120833,125 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ -static void fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ + int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( *p | c ) c = *p++ & 0x80; - p++; - - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } + assert( p ); - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } }else{ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } } + + return SQLITE_OK; } -/* -** This function is called to estimate the amount of data that will be -** loaded from the disk If SegReaderIterate() is called on this seg-reader, -** in units of average document size. -** -** This can be used as follows: If the caller has a small doclist that -** contains references to N documents, and is considering merging it with -** a large doclist (size X "average documents"), it may opt not to load -** the large doclist if X>N. -*/ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCost( - Fts3Cursor *pCsr, /* FTS3 cursor handle */ - Fts3SegReader *pReader, /* Segment-reader handle */ - int *pnCost /* IN/OUT: Number of bytes read */ + +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl ){ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Return code */ - int nCost = 0; /* Cost in bytes to return */ - int pgsz = p->nPgsz; /* Database page size */ + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; - /* If this seg-reader is reading the pending-terms table, or if all data - ** for the segment is stored on the root page of the b-tree, then the cost - ** is zero. In this case all required data is already in main memory. - */ - if( p->bHasStat - && !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - int nBlob = 0; - sqlite3_int64 iBlock; + assert( p->bHasStat ); + assert( pgsz>0 ); - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - sqlite3_stmt *pStmt; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); - if( rc ) return rc; - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *a = sqlite3_column_blob(pStmt, 0); - if( a ){ - const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; } - - pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz); - } - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc; - } - - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - int nOvfl = (nBlob + 34)/pgsz; - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); } } } - - *pnCost += nCost; + *pnOvfl = nOvfl; return rc; } @@ -115551,6 +120964,7 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); } } sqlite3_free(pReader); @@ -115627,24 +121041,42 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ /* ** This function is used to allocate an Fts3SegReader that iterates through ** a subset of the terms stored in the Fts3Table.pendingTerms array. +** +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): +** +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite +** +** Whereas if isPrefixIter is zero, the terms visited are: +** +** firebird mysql sqlite */ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isPrefix, /* True for a term-prefix query */ + int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; - if( isPrefix ){ + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ @@ -115661,6 +121093,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } aElem = aElem2; } + aElem[nElem++] = pE; } } @@ -115674,7 +121107,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } }else{ - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. */ + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -115694,49 +121129,13 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } } - if( isPrefix ){ + if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; return rc; } - -/* -** The second argument to this function is expected to be a statement of -** the form: -** -** SELECT -** idx, -- col 0 -** start_block, -- col 1 -** leaves_end_block, -- col 2 -** end_block, -- col 3 -** root -- col 4 -** FROM %_segdir ... -** -** This function allocates and initializes a Fts3SegReader structure to -** iterate through the terms stored in the segment identified by the -** current row that pStmt is pointing to. -** -** If successful, the Fts3SegReader is left pointing to the first term -** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error -** code is returned. -*/ -static int fts3SegReaderNew( - sqlite3_stmt *pStmt, /* See above */ - int iAge, /* Segment "age". */ - Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ -){ - return sqlite3Fts3SegReaderNew(iAge, - sqlite3_column_int64(pStmt, 1), - sqlite3_column_int64(pStmt, 2), - sqlite3_column_int64(pStmt, 3), - sqlite3_column_blob(pStmt, 4), - sqlite3_column_bytes(pStmt, 4), - ppReader - ); -} - /* ** Compare the entries pointed to by two Fts3SegReader structures. ** Comparison is as follows: @@ -115794,6 +121193,18 @@ static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ assert( pLhs->aNode && pRhs->aNode ); return rc; } +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} /* ** Compare the term that the Fts3SegReader object passed as the first argument @@ -116322,16 +121733,16 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ ** The first value in the apVal[] array is assumed to contain an integer. ** This function tests if there exist any documents with docid values that ** are different from that integer. i.e. if deleting the document with docid -** apVal[0] would mean the FTS3 table were empty. +** pRowid would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for -** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ -static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pisEmpty = sqlite3_column_int(pStmt, 0); @@ -116342,40 +121753,30 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ } /* -** Set *pnSegment to the number of segments of level iLevel in the database. +** Set *pnMax to the largest segment level in the database for the index +** iIndex. ** -** Return SQLITE_OK if successful, or an SQLite error code if not. -*/ -static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){ - sqlite3_stmt *pStmt; - int rc; - - assert( iLevel>=0 ); - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, iLevel); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - } - return sqlite3_reset(pStmt); -} - -/* -** Set *pnSegment to the total number of segments in the database. Set -** *pnMax to the largest segment level in the database (segment levels -** are stored in the 'level' column of the %_segdir table). +** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; + assert( iIndex>=0 && iIndexnIndex ); - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - *pnMax = sqlite3_column_int(pStmt, 1); + *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -116396,6 +121797,7 @@ static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ @@ -116418,15 +121820,23 @@ static int fts3DeleteSegdir( return rc; } - if( iLevel>=0 ){ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iLevel); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } }else{ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } return rc; @@ -116476,84 +121886,105 @@ static void fts3ColumnFilter( } /* -** sqlite3Fts3SegReaderIterate() callback used when merging multiple -** segments to create a single, larger segment. +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. */ -static int fts3MergeCallback( - Fts3Table *p, /* FTS3 Virtual table handle */ - void *pContext, /* Pointer to SegmentWriter* to write with */ - char *zTerm, /* Term to write to the db */ - int nTerm, /* Number of bytes in zTerm */ - char *aDoclist, /* Doclist associated with zTerm */ - int nDoclist /* Number of bytes in doclist */ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList ){ - SegmentWriter **ppW = (SegmentWriter **)pContext; - return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist); -} + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } -/* -** sqlite3Fts3SegReaderIterate() callback used when flushing the contents -** of the pending-terms hash table to the database. -*/ -static int fts3FlushCallback( - Fts3Table *p, /* FTS3 Virtual table handle */ - void *pContext, /* Pointer to SegmentWriter* to write with */ - char *zTerm, /* Term to write to the db */ - int nTerm, /* Number of bytes in zTerm */ - char *aDoclist, /* Doclist associated with zTerm */ - int nDoclist /* Number of bytes in doclist */ -){ - SegmentWriter **ppW = (SegmentWriter **)pContext; - return fts3SegWriterAdd(p, ppW, 0, zTerm, nTerm, aDoclist, nDoclist); + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; } -/* -** This function is used to iterate through a contiguous set of terms -** stored in the full-text index. It merges data contained in one or -** more segments to support this. -** -** The second argument is passed an array of pointers to SegReader objects -** allocated with sqlite3Fts3SegReaderNew(). This function merges the range -** of terms selected by each SegReader. If a single term is present in -** more than one segment, the associated doclists are merged. For each -** term and (possibly merged) doclist in the merged range, the callback -** function xFunc is invoked with its arguments set as follows. -** -** arg 0: Copy of 'p' parameter passed to this function -** arg 1: Copy of 'pContext' parameter passed to this function -** arg 2: Pointer to buffer containing term -** arg 3: Size of arg 2 buffer in bytes -** arg 4: Pointer to buffer containing doclist -** arg 5: Size of arg 2 buffer in bytes -** -** The 4th argument to this function is a pointer to a structure of type -** Fts3SegFilter, defined in fts3Int.h. The contents of this structure -** further restrict the range of terms that callbacks are made for and -** modify the behaviour of this function. See comments above structure -** definition for details. -*/ -SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( Fts3Table *p, /* Virtual table handle */ - Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */ - int nSegment, /* Size of apSegment array */ - Fts3SegFilter *pFilter, /* Restrictions on range of iteration */ - int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */ - void *pContext /* Callback context (2nd argument) */ -){ - int i; /* Iterator variable */ - char *aBuffer = 0; /* Buffer to merge doclists in */ - int nAlloc = 0; /* Allocated size of aBuffer buffer */ - int rc = SQLITE_OK; /* Return code */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } - int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); - int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); - int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); - int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; - /* If there are zero segments, this function is a no-op. This scenario - ** comes about only when reading from an empty database. - */ - if( nSegment==0 ) goto finished; + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + } + + if( nList>0 ){ + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + *paPoslist = pMsr->aBuffer; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + }else{ + *paPoslist = pList; + } + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of @@ -116561,21 +121992,143 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - for(i=0; inTerm; - const char *zTerm = pFilter->zTerm; - Fts3SegReader *pSeg = apSegment[i]; + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - rc = fts3SegReaderNext(p, pSeg); - if( rc!=SQLITE_OK ) goto finished; + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); - fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp); - while( apSegment[0]->aNode ){ - int nTerm = apSegment[0]->nTerm; - char *zTerm = apSegment[0]->zTerm; - int nMerge = 1; + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderNext(p, apSegment[i], 0); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; + + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; /* If this is a prefix-search, and if the term that apSegment[0] points ** to does not share a suffix with pFilter->zTerm/nTerm, then all @@ -116584,53 +122137,62 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( ** Similarly, if this is a search for an exact match, and the first term ** of segment apSegment[0] is not a match, exit early. */ - if( pFilter->zTerm ){ - if( nTermnTerm - || (!isPrefix && nTerm>pFilter->nTerm) - || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) - ){ - goto finished; + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; } } + nMerge = 1; while( nMergeaNode - && apSegment[nMerge]->nTerm==nTerm - && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm) + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) ){ nMerge++; } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ - Fts3SegReader *p0 = apSegment[0]; - rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist); - if( rc!=SQLITE_OK ) goto finished; + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ /* The current term of the first nMerge entries in the array ** of Fts3SegReader objects is the same. The doclists must be merged - ** and a single term added to the new segment. + ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } @@ -116639,53 +122201,67 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); - if( nDoclist+nByte>nAlloc ){ + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; - nAlloc = nDoclist+nByte*2; - aNew = sqlite3_realloc(aBuffer, nAlloc); + pCsr->nBuffer = (nDoclist+nByte)*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ - rc = SQLITE_NOMEM; - goto finished; + return SQLITE_NOMEM; } - aBuffer = aNew; + pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ - memcpy(&aBuffer[nDoclist], pList, nList); + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); nDoclist += nList; - aBuffer[nDoclist++] = '\0'; + pCsr->aBuffer[nDoclist++] = '\0'; } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } - if( nDoclist>0 ){ - rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist); - if( rc!=SQLITE_OK ) goto finished; + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; } } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); - /* If there is a term specified to filter on, and this is not a prefix - ** search, return now. The callback that corresponds to the required - ** term (if such a term exists in the index) has already been made. - */ - if( pFilter->zTerm && !isPrefix ){ - goto finished; - } + return rc; +} - for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); } - fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp); - } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); - finished: - sqlite3_free(aBuffer); - return rc; + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } } /* @@ -116699,157 +122275,91 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ - int i; /* Iterator variable */ +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ - int iIdx; /* Index of new segment */ - int iNewLevel = 0; /* Level to create new segment at */ - sqlite3_stmt *pStmt = 0; - SegmentWriter *pWriter = 0; - int nSegment = 0; /* Number of segments being merged */ - Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ - Fts3SegReader *pPending = 0; /* Iterator for pending-terms */ + int iIdx = 0; /* Index of new segment */ + int iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ - if( iLevel<0 ){ + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically - ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. - */ - iIdx = 0; - rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel); - if( rc!=SQLITE_OK ) goto finished; - nSegment += (pPending!=0); - if( nSegment<=1 ){ - return SQLITE_DONE; + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 ){ + rc = SQLITE_DONE; + goto finished; } + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + bIgnoreEmpty = 1; + + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ - /* This call is to merge all segments at level iLevel. Find the next + /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to - ** a single iLevel+2 segment if necessary. - */ - iNewLevel = iLevel+1; - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegmentCount(p, iLevel, &nSegment); - if( rc!=SQLITE_OK ) goto finished; - } - assert( nSegment>0 ); - assert( iNewLevel>=0 ); - - /* Allocate space for an array of pointers to segment iterators. */ - apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment); - if( !apSegment ){ - rc = SQLITE_NOMEM; - goto finished; + ** a single iLevel+2 segment if necessary. */ + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } - memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment); - - /* Allocate a Fts3SegReader structure for each segment being merged. A - ** Fts3SegReader stores the state data required to iterate through all - ** entries on all leaves of a single segment. - */ - assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL); - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0); - if( rc!=SQLITE_OK ) goto finished; - sqlite3_bind_int(pStmt, 1, iLevel); - for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){ - rc = fts3SegReaderNew(pStmt, i, &apSegment[i]); - if( rc!=SQLITE_OK ){ - goto finished; - } - } - rc = sqlite3_reset(pStmt); - if( pPending ){ - apSegment[i] = pPending; - pPending = 0; - } - pStmt = 0; if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0); - rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, - &filter, fts3MergeCallback, (void *)&pWriter - ); + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + while( SQLITE_OK==rc ){ + rc = sqlite3Fts3SegReaderStep(p, &csr); + if( rc!=SQLITE_ROW ) break; + rc = fts3SegWriterAdd(p, &pWriter, 1, + csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); + } if( rc!=SQLITE_OK ) goto finished; + assert( pWriter ); - rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment); - if( rc==SQLITE_OK ){ - rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); + if( rc!=SQLITE_OK ) goto finished; } + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: fts3SegWriterFree(pWriter); - if( apSegment ){ - for(i=0; inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } + sqlite3Fts3PendingTermsClear(p); return rc; } @@ -117001,6 +122511,23 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int i; + int bSeenDone = 0; + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -117017,12 +122544,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, -1); - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - }else{ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -117035,44 +122557,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } - sqlite3Fts3SegmentsClose(p); return rc; } -/* -** Return the deferred doclist associated with deferred token pDeferred. -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already -** been called to allocate and populate the doclist. -*/ -SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ - if( pDeferred->pList ){ - *pnByte = pDeferred->pList->nData; - return pDeferred->pList->aData; - } - *pnByte = 0; - return 0; -} - -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - if( pExpr->isLoaded ){ - sqlite3_free(pExpr->aDoclist); - pExpr->isLoaded = 0; - pExpr->aDoclist = 0; - pExpr->nDoclist = 0; - pExpr->pCurrent = 0; - pExpr->iCurrent = 0; - } - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -117080,12 +122567,9 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* @@ -117097,7 +122581,7 @@ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; @@ -117162,6 +122646,33 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ return rc; } +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ @@ -117187,6 +122698,40 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( return SQLITE_OK; } +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnDoc, + u32 *aSzDel +){ + int isEmpty = 0; + int rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p); + *pnDoc = *pnDoc - 1; + }else{ + sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); + rc = fts3PendingTermsDocid(p, iRemove); + fts3DeleteTerms(&rc, p, pRowid, aSzDel); + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + + return rc; +} /* ** This function does the work for the xUpdate method of FTS3 virtual @@ -117202,49 +122747,97 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; assert( p->pSegments==0 ); + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); - /* If this is a DELETE or UPDATE operation, remove the old record. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty = 0; - rc = fts3IsEmpty(p, apVal, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. - */ - rc = fts3DeleteAll(p); + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); }else{ - isRemove = 1; - iRemove = sqlite3_value_int64(apVal[0]); - rc = fts3PendingTermsDocid(p, iRemove); - fts3DeleteTerms(&rc, p, apVal, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); - } - nChng--; + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; } } - }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - sqlite3_free(aSzIns); - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; + iRemove = sqlite3_value_int64(apVal[0]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ - rc = fts3InsertData(p, apVal, pRowid); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB; + } if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ rc = fts3PendingTermsDocid(p, *pRowid); } @@ -117261,6 +122854,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; @@ -117275,12 +122869,10 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, -1); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); @@ -117469,92 +123061,24 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } -/* -** The argument to this function is always a phrase node. Its doclist -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes -** to the left of this one in the query tree have already been loaded. -** -** If this phrase node is part of a series of phrase nodes joined by -** NEAR operators (and is not the left-most of said series), then elements are -** removed from the phrases doclist consistent with the NEAR restriction. If -** required, elements may be removed from the doclists of phrases to the -** left of this one that are part of the same series of NEAR operator -** connected phrases. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ExprNearTrim(Fts3Expr *pExpr){ - int rc = SQLITE_OK; - Fts3Expr *pParent = pExpr->pParent; - - assert( pExpr->eType==FTSQUERY_PHRASE ); - while( rc==SQLITE_OK - && pParent - && pParent->eType==FTSQUERY_NEAR - && pParent->pRight==pExpr - ){ - /* This expression (pExpr) is the right-hand-side of a NEAR operator. - ** Find the expression to the left of the same operator. - */ - int nNear = pParent->nNear; - Fts3Expr *pLeft = pParent->pLeft; - - if( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pRight; - } - - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); - - pExpr = pLeft; - pParent = pExpr->pParent; - } - - return rc; -} - /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also ** fts3ExprLoadDoclists(). */ -static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; UNUSED_PARAMETER(iPhrase); p->nPhrase++; - p->nToken += pExpr->pPhrase->nToken; - - if( pExpr->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pExpr->isLoaded = 1; - if( rc==SQLITE_OK ){ - rc = fts3ExprNearTrim(pExpr); - } - } + p->nToken += pPhrase->nToken; return rc; } -/* -** This is an fts3ExprIterate() callback used while loading the doclists -** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also -** fts3ExprLoadDoclists(). -*/ -static int fts3ExprLoadDoclistsCb2(Fts3Expr *pExpr, int iPhrase, void *ctx){ - UNUSED_PARAMETER(iPhrase); - UNUSED_PARAMETER(ctx); - if( pExpr->aDoclist ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent, &pExpr->iCurrent); - } - return SQLITE_OK; -} - /* ** Load the doclists for each phrase in the query associated with FTS3 cursor ** pCsr. @@ -117573,10 +123097,7 @@ static int fts3ExprLoadDoclists( int rc; /* Return Code */ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx); - if( rc==SQLITE_OK ){ - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0); - } + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; @@ -117727,7 +123248,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -118084,26 +123605,6 @@ static int fts3ColumnlistCount(char **ppCollist){ return nEntry; } -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ - char *pCsr = *pp; - while( *pCsr ){ - int nHit; - sqlite3_int64 iCol = 0; - if( *pCsr==0x01 ){ - pCsr++; - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); - } - nHit = fts3ColumnlistCount(&pCsr); - assert( nHit>0 ); - if( isGlobal ){ - aOut[iCol*3+1]++; - } - aOut[iCol*3] += nHit; - } - pCsr++; - *pp = pCsr; -} - /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -118137,48 +123638,9 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - Fts3Cursor *pCsr = p->pCursor; - char *pIter; - char *pEnd; - char *pFree = 0; - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - - assert( pExpr->isLoaded ); - assert( pExpr->eType==FTSQUERY_PHRASE ); - - if( pCsr->pDeferred ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - for(ii=0; iinToken; ii++){ - if( pPhrase->aToken[ii].bFulltext ) break; - } - if( iinToken ){ - int nFree = 0; - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); - if( rc!=SQLITE_OK ) return rc; - pIter = pFree; - pEnd = &pFree[nFree]; - }else{ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - return SQLITE_OK; - } - }else{ - pIter = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - } - - /* Fill in the global hit count matrix row for this phrase. */ - while( pIterpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* @@ -118192,17 +123654,16 @@ static int fts3ExprLocalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; - if( pExpr->aDoclist ){ + for(i=0; inCol; i++){ char *pCsr; - int iStart = iPhrase * p->nCol * 3; - int i; - - for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); if( pCsr ){ - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } } @@ -118268,9 +123729,11 @@ static int fts3MatchinfoSelectDoctotal( if( rc!=SQLITE_OK ) return rc; } pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); + if( nDoc==0 ) return SQLITE_CORRUPT_VTAB; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; @@ -118286,9 +123749,8 @@ static int fts3MatchinfoSelectDoctotal( typedef struct LcsIterator LcsIterator; struct LcsIterator { Fts3Expr *pExpr; /* Pointer to phrase expression */ - char *pRead; /* Cursor used to iterate through aDoclist */ int iPosOffset; /* Tokens count up to end of this phrase */ - int iCol; /* Current column number */ + char *pRead; /* Cursor used to iterate through aDoclist */ int iPos; /* Current position */ }; @@ -118319,17 +123781,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ int rc = 0; pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 ){ - pIter->iCol = LCS_ITERATOR_FINISHED; + if( iRead==0 || iRead==1 ){ + pRead = 0; rc = 1; }else{ - if( iRead==1 ){ - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - pIter->iCol = (int)iRead; - pIter->iPos = pIter->iPosOffset; - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - rc = 1; - } pIter->iPos += (int)(iRead-2); } @@ -118361,42 +123816,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1); - if( pIter->pRead ){ - pIter->iPos = pIter->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - }else{ - pIter->iCol = LCS_ITERATOR_FINISHED; - } } for(iCol=0; iColnCol; iCol++){ int nLcs = 0; /* LCS value for this column */ int nLive = 0; /* Number of iterators in aIter not at EOF */ - /* Loop through the iterators in aIter[]. Set nLive to the number of - ** iterators that point to a position-list corresponding to column iCol. - */ for(i=0; inPhrase; i++){ - assert( aIter[i].iCol>=iCol ); - if( aIter[i].iCol==iCol ) nLive++; + LcsIterator *pIt = &aIter[i]; + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } } - /* The following loop runs until all iterators in aIter[] have finished - ** iterating through positions in column iCol. Exactly one of the - ** iterators is advanced each time the body of the loop is run. - */ while( nLive>0 ){ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ int nThisLcs = 0; /* LCS for the current iterator positions */ for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; - if( iCol!=pIter->iCol ){ + if( pIter->pRead==0 ){ /* This iterator is already at EOF for this column. */ nThisLcs = 0; }else{ @@ -118462,7 +123909,7 @@ static int fts3MatchinfoValues( case FTS3_MATCHINFO_NDOC: if( bGlobal ){ - sqlite3_int64 nDoc; + sqlite3_int64 nDoc = 0; rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); pInfo->aMatchinfo[0] = (u32)nDoc; } @@ -118477,9 +123924,11 @@ static int fts3MatchinfoValues( if( rc==SQLITE_OK ){ int iCol; for(iCol=0; iColnCol; iCol++){ + u32 iVal; sqlite3_int64 nToken; a += sqlite3Fts3GetVarint(a, &nToken); - pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; } } } @@ -118716,6 +124165,7 @@ struct TermOffset { }; struct TermOffsetCtx { + Fts3Cursor *pCsr; int iCol; /* Column of table to populate aTerm for */ int iTerm; sqlite3_int64 iDocid; @@ -118733,7 +124183,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); @@ -118786,6 +124236,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( goto offsets_out; } sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; /* Loop through the table columns, appending offset information to ** string-buffer res for each column. @@ -118861,7 +124312,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( ); rc = fts3StringAppend(&res, aBuffer, -1); }else if( rc==SQLITE_DONE ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } } @@ -119449,17 +124900,17 @@ nodeAcquire( if( pNode && iNode==1 ){ pRtree->iDepth = readInt16(pNode->zData); if( pRtree->iDepth>RTREE_MAX_DEPTH ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } /* If no error has occurred so far, check if the "number of entries" ** field on the node is too large. If so, set the return code to - ** SQLITE_CORRUPT. + ** SQLITE_CORRUPT_VTAB. */ if( pNode && rc==SQLITE_OK ){ if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } @@ -119467,7 +124918,7 @@ nodeAcquire( if( pNode!=0 ){ nodeHashInsert(pRtree, pNode); }else{ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } *ppNode = pNode; }else{ @@ -119994,7 +125445,7 @@ static int nodeRowidIndex( return SQLITE_OK; } } - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } /* @@ -120200,7 +125651,7 @@ static int rtreeFilter( rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); - assert( (idxStr==0 && argc==0) || strlen(idxStr)==argc*2 ); + assert( (idxStr==0 && argc==0) || (int)strlen(idxStr)==argc*2 ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; @@ -120285,7 +125736,7 @@ static int rtreeFilter( */ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int rc = SQLITE_OK; - int ii, cCol; + int ii; int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; @@ -120293,7 +125744,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ UNUSED_PARAMETER(tab); assert( pIdxInfo->idxStr==0 ); - for(ii=0; iinConstraint; ii++){ + for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; if( p->usable && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ @@ -120317,9 +125768,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ } if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ - int j, opmsk; - static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; - u8 op = 0; + u8 op; switch( p->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; @@ -120331,37 +125780,10 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ op = RTREE_MATCH; break; } - assert( op!=0 ); - - /* Make sure this particular constraint has not been used before. - ** If it has been used before, ignore it. - ** - ** A <= or < can be used if there is a prior >= or >. - ** A >= or > can be used if there is a prior < or <=. - ** A <= or < is disqualified if there is a prior <=, <, or ==. - ** A >= or > is disqualified if there is a prior >=, >, or ==. - ** A == is disqualifed if there is any prior constraint. - */ - assert( compatible[RTREE_EQ & 7]==0 ); - assert( compatible[RTREE_LT & 7]==1 ); - assert( compatible[RTREE_LE & 7]==1 ); - assert( compatible[RTREE_GT & 7]==2 ); - assert( compatible[RTREE_GE & 7]==2 ); - cCol = p->iColumn - 1 + 'a'; - opmsk = compatible[op & 7]; - for(j=0; jaConstraintUsage[ii].argvIndex = (iIdx/2); - pIdxInfo->aConstraintUsage[ii].omit = 1; - } + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = p->iColumn - 1 + 'a'; + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; } } @@ -120382,7 +125804,7 @@ static float cellArea(Rtree *pRtree, RtreeCell *p){ float area = 1.0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + area = (float)(area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } return area; } @@ -120395,7 +125817,7 @@ static float cellMargin(Rtree *pRtree, RtreeCell *p){ float margin = 0.0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + margin += (float)(DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } return margin; } @@ -120480,7 +125902,7 @@ static float cellOverlap( o = 0.0; break; }else{ - o = o * (x2-x1); + o = o * (float)(x2-x1); } } overlap += o; @@ -120499,12 +125921,12 @@ static float cellOverlapEnlargement( int nCell, int iExclude ){ - float before; - float after; + double before; + double after; before = cellOverlap(pRtree, p, aCell, nCell, iExclude); cellUnion(pRtree, p, pInsert); after = cellOverlap(pRtree, p, aCell, nCell, iExclude); - return after-before; + return (float)(after-before); } #endif @@ -120526,11 +125948,11 @@ static int ChooseLeaf( for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; - sqlite3_int64 iBest; + sqlite3_int64 iBest = 0; - float fMinGrowth; - float fMinArea; - float fMinOverlap; + float fMinGrowth = 0.0; + float fMinArea = 0.0; + float fMinOverlap = 0.0; int nCell = NCELL(pNode); RtreeCell cell; @@ -120618,7 +126040,7 @@ static int AdjustTree( int iCell; if( nodeParentIndex(pRtree, p, &iCell) ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } nodeGetCell(pRtree, pParent, iCell, &cell); @@ -120960,9 +126382,9 @@ static int splitNodeStartree( int *aSpare; int ii; - int iBestDim; - int iBestSplit; - float fBestMargin; + int iBestDim = 0; + int iBestSplit = 0; + float fBestMargin = 0.0; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -120984,9 +126406,9 @@ static int splitNodeStartree( for(ii=0; iinDim; ii++){ float margin = 0.0; - float fBestOverlap; - float fBestArea; - int iBestLeft; + float fBestOverlap = 0.0; + float fBestArea = 0.0; + int iBestLeft = 0; int nLeft; for( @@ -121290,7 +126712,7 @@ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ } rc = sqlite3_reset(pRtree->pReadParent); if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; pChild = pChild->pParent; } return rc; @@ -121301,7 +126723,7 @@ static int deleteCell(Rtree *, RtreeNode *, int, int); static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ int rc; int rc2; - RtreeNode *pParent; + RtreeNode *pParent = 0; int iCell; assert( pNode->nRef==1 ); @@ -121449,19 +126871,19 @@ static int Reinsert( } aOrder[ii] = ii; for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2+1]); } } for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = aCenterCoord[iDim]/((float)nCell*2.0); + aCenterCoord[iDim] = (float)(aCenterCoord[iDim]/((float)nCell*2.0)); } for(ii=0; iinDim; iDim++){ - float coord = DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2]); + float coord = (float)(DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); } } @@ -121560,10 +126982,10 @@ static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ /* Find a node to store this cell in. pNode->iNode currently contains ** the height of the sub-tree headed by the cell. */ - rc = ChooseLeaf(pRtree, &cell, pNode->iNode, &pInsert); + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); if( rc==SQLITE_OK ){ int rc2; - rc = rtreeInsertCell(pRtree, pInsert, &cell, pNode->iNode); + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); rc2 = nodeRelease(pRtree, pInsert); if( rc==SQLITE_OK ){ rc = rc2; @@ -121587,113 +127009,119 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ } /* -** The xUpdate method for rtree module virtual tables. +** Remove the entry with rowid=iDelete from the r-tree structure. */ -static int rtreeUpdate( - sqlite3_vtab *pVtab, - int nData, - sqlite3_value **azData, - sqlite_int64 *pRowid -){ - Rtree *pRtree = (Rtree *)pVtab; - int rc = SQLITE_OK; +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ - rtreeReference(pRtree); - assert(nData>=1); + /* Obtain a reference to the root node to initialise Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); - /* If azData[0] is not an SQL NULL value, it is the rowid of a - ** record to delete from the r-tree table. The following block does - ** just that. + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. */ - if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ - i64 iDelete; /* The rowid to delete */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ - int iCell; /* Index of iDelete cell in pLeaf */ - RtreeNode *pRoot; - - /* Obtain a reference to the root node to initialise Rtree.iDepth */ - rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf); + } - /* Obtain a reference to the leaf node that contains the entry - ** about to be deleted. - */ + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); if( rc==SQLITE_OK ){ - iDelete = sqlite3_value_int64(azData[0]); - rc = findLeafNode(pRtree, iDelete, &pLeaf); + rc = deleteCell(pRtree, pLeaf, iCell, 0); } - - /* Delete the cell in question from the leaf node. */ + rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ - int rc2; - rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); - if( rc==SQLITE_OK ){ - rc = deleteCell(pRtree, pLeaf, iCell, 0); - } - rc2 = nodeRelease(pRtree, pLeaf); - if( rc==SQLITE_OK ){ - rc = rc2; - } + rc = rc2; } + } - /* Delete the corresponding entry in the _rowid table. */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); - sqlite3_step(pRtree->pDeleteRowid); - rc = sqlite3_reset(pRtree->pDeleteRowid); - } + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } - /* Check if the root node now has exactly one child. If so, remove - ** it, schedule the contents of the child for reinsertion and - ** reduce the tree height by one. - ** - ** This is equivalent to copying the contents of the child into - ** the root node (the operation that Gutman's paper says to perform - ** in this scenario). - */ - if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ - int rc2; - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - rc2 = nodeRelease(pRtree, pChild); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); } - - /* Re-insert the contents of any underfull nodes removed from the tree. */ - for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ - if( rc==SQLITE_OK ){ - rc = reinsertNodeContent(pRtree, pLeaf); - } - pRtree->pDeleted = pLeaf->pNext; - sqlite3_free(pLeaf); + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; } + } - /* Release the reference to the root node. */ + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRoot); - }else{ - nodeRelease(pRtree, pRoot); + rc = reinsertNodeContent(pRtree, pLeaf); } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); } - /* If the azData[] array contains more than one element, elements - ** (azData[2]..azData[argc-1]) contain a new record to insert into - ** the r-tree structure. + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} + +/* +** The xUpdate method for rtree module virtual tables. +*/ +static int rtreeUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **azData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ + + rtreeReference(pRtree); + assert(nData>=1); + + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. */ - if( rc==SQLITE_OK && nData>1 ){ - /* Insert a new record into the r-tree */ - RtreeCell cell; + if( nData>1 ){ int ii; - RtreeNode *pLeaf; /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ assert( nData==(pRtree->nDim*2 + 3) ); @@ -121717,18 +127145,49 @@ static int rtreeUpdate( } } - /* Figure out the rowid of the new row. */ - if( sqlite3_value_type(azData[2])==SQLITE_NULL ){ - rc = newRowid(pRtree, &cell.iRowid); - }else{ + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ cell.iRowid = sqlite3_value_int64(azData[2]); - sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); - if( SQLITE_ROW==sqlite3_step(pRtree->pReadRowid) ){ - sqlite3_reset(pRtree->pReadRowid); - rc = SQLITE_CONSTRAINT; - goto constraint; + if( sqlite3_value_type(azData[0])==SQLITE_NULL + || sqlite3_value_int64(azData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } } - rc = sqlite3_reset(pRtree->pReadRowid); + bHaveRowid = 1; + } + } + + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + } + + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; @@ -121773,7 +127232,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ } static sqlite3_module rtreeModule = { - 0, /* iVersion */ + 0, /* iVersion */ rtreeCreate, /* xCreate - create a table */ rtreeConnect, /* xConnect - connect to an existing table */ rtreeBestIndex, /* xBestIndex - Determine search strategy */ @@ -121792,7 +127251,10 @@ static sqlite3_module rtreeModule = { 0, /* xCommit - commit transaction */ 0, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ - rtreeRename /* xRename - rename the table */ + rtreeRename, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; static int rtreeSqlInit( @@ -121912,7 +127374,7 @@ static int getNodeSize( int rc; char *zSql; if( isCreate ){ - int iPageSize; + int iPageSize = 0; zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); rc = getIntFromStmt(db, zSql, &iPageSize); if( rc==SQLITE_OK ){ @@ -121969,6 +127431,8 @@ static int rtreeInit( return SQLITE_ERROR; } + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); @@ -122065,7 +127529,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ int jj; nodeGetCell(&tree, &node, ii, &cell); - sqlite3_snprintf(512-nCell,&zCell[nCell],"%d", cell.iRowid); + sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); nCell = strlen(zCell); for(jj=0; jjzName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 @@ -122711,10 +128177,7 @@ SQLITE_API int sqlite3_extension_init( ** ************************************************************************* ** This file implements a tokenizer for fts3 based on the ICU library. -** -** $Id: sqlite3.c 306223 2010-12-11 14:57:34Z iliaa $ */ - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU -- cgit v1.2.3